2-hyperparameter_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning using HyperDrive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import json\n",
    "import pathlib\n",
    "import requests\n",
    "\n",
    "import joblib\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import azureml.core\n",
    "from azureml.core import Workspace, Experiment\n",
    "from azureml.core.compute import ComputeTarget\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "from azureml.core.dataset import Dataset\n",
    "from azureml.widgets import RunDetails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "gather": {
     "logged": 1598531914256
    }
   },
   "outputs": [],
   "source": [
    "from nd00333.model.hyperdrive.train import run_config as hd_train_config\n",
    "from nd00333.model.deploy import run_config as deploy_config\n",
    "from nd00333 import utils as package_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "azureml.core.VERSION 1.21.0\n"
     ]
    }
   ],
   "source": [
    "print(\"azureml.core.VERSION\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an experiment\n",
    "\n",
    "Create an experiment identified by the creation date. The purpose of identifying the experiments is to not mix the manually run experiments (using this jupyter notebook) with the experiments run using the deployment automation of the master git branch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(level=logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!az logout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "gather": {
     "logged": 1598531917374
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n"
     ]
    }
   ],
   "source": [
    "workspace = Workspace.from_config()\n",
    "utcnow = datetime.datetime.utcnow().strftime(\"%Y-%m-%d-%H-%m\")\n",
    "utcnow = \"2021-01-29-09-01\"\n",
    "experiment_name = f\"jupyter-{utcnow}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Created a worker pool for first use\"}\n",
      "{\"message\": \"Created a worker pool for first use\"}\n",
      "{\"message\": \"Created a worker pool for first use\"}\n",
      "{\"message\": \"Created a worker pool for first use\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "experiment_name jupyter-2021-01-29-09-01\n"
     ]
    }
   ],
   "source": [
    "experiment = Experiment(workspace, experiment_name)\n",
    "print(f\"experiment_name {experiment_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "\n",
    "Load the dataset from local files on disk into the default AzureML workspace datastore.\n",
    "Store the versioned (by the \"_1\" suffix) train, validate and test sets under separate datastore paths. The uploading of large datasets tends to break with \"The write operation timed out\" when using the Python SDK and therefore the `az` is used. See https://docs.microsoft.com/en-us/answers/questions/43980/cannot-upload-local-files-to-azureml-datastore-pyt.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_train_name = \"ids2018train_1\"\n",
    "dataset_validate_name = \"ids2018validate_1\"\n",
    "dataset_test_name = \"ids2018test_1\"\n",
    "dataset_2017_name = \"ids2017full_1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mTo sign in, use a web browser to open the page https://microsoft.com/devicelogin and enter the code D85M4TGUT to authenticate.\u001b[0m\n",
      "[\n",
      "  {\n",
      "    \"cloudName\": \"AzureCloud\",\n",
      "    \"homeTenantId\": \"660b3398-b80e-49d2-bc5b-ac1dc93b5254\",\n",
      "    \"id\": \"2c48c51c-bd47-40d4-abbe-fb8eabd19c8c\",\n",
      "    \"isDefault\": true,\n",
      "    \"managedByTenants\": [],\n",
      "    \"name\": \"Udacity CloudLabs Sub - 03\",\n",
      "    \"state\": \"Enabled\",\n",
      "    \"tenantId\": \"660b3398-b80e-49d2-bc5b-ac1dc93b5254\",\n",
      "    \"user\": {\n",
      "      \"name\": \"odl_user_136531@udacitylabs.onmicrosoft.com\",\n",
      "      \"type\": \"user\"\n",
      "    }\n",
      "  }\n",
      "]\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mExtension 'azure-cli-ml' is already installed.\u001b[0m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az extension add --name azure-cli-ml --version 1.21.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "azure-cli                         2.18.0\n",
      "\n",
      "core                              2.18.0\n",
      "telemetry                          1.0.6\n",
      "\n",
      "Extensions:\n",
      "azure-cli-ml                      1.21.0\n",
      "\n",
      "Python location '/opt/az/bin/python3'\n",
      "Extensions directory '/home/vscode/.azure/cliextensions'\n",
      "\n",
      "Python (Linux) 3.6.10 (default, Jan 15 2021, 09:54:35) \n",
      "[GCC 8.3.0]\n",
      "\n",
      "Legal docs and information: aka.ms/AzureCliLegal\n",
      "\n",
      "\n",
      "Your CLI is up-to-date.\n",
      "\n",
      "\u001b[33m\u001b[1mPlease let us know how we are doing: \u001b[34mhttps://aka.ms/azureclihats\u001b[0m\n",
      "\u001b[33m\u001b[1mand let us know if you're interested in trying out our newest features: \u001b[34mhttps://aka.ms/CLIUXstudy\u001b[0m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 1 files\n",
      "Uploading datasets/ids2018train/data.csv\n",
      "Uploaded datasets/ids2018train/data.csv, 1 files out of an estimated total of 1\n",
      "Uploaded 1 files\n",
      "\u001b[32mCommand ran in 1160.690 seconds (init: 0.226, invoke: 1160.464)\u001b[0m\n",
      "\u001b[0mCPU times: user 1min 6s, sys: 11.8 s, total: 1min 17s\n",
      "Wall time: 19min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2018train --target-path \\\n",
    "    $dataset_train_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 1 files\n",
      "Uploading datasets/ids2018validate/data.csv\n",
      "Uploaded datasets/ids2018validate/data.csv, 1 files out of an estimated total of 1\n",
      "Uploaded 1 files\n",
      "\u001b[32mCommand ran in 238.429 seconds (init: 0.305, invoke: 238.124)\u001b[0m\n",
      "\u001b[0mCPU times: user 13.9 s, sys: 2.6 s, total: 16.5 s\n",
      "Wall time: 3min 59s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2018validate --target-path \\\n",
    "    $dataset_validate_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 1 files\n",
      "Target already exists. Skipping upload for ids2018test_1/data.csv\n",
      "Uploaded 0 files\n",
      "\u001b[32mCommand ran in 6.275 seconds (init: 0.273, invoke: 6.002)\u001b[0m\n",
      "\u001b[0mCPU times: user 325 ms, sys: 41.6 ms, total: 367 ms\n",
      "Wall time: 7.73 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2018test --target-path \\\n",
    "    $dataset_test_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 8 files\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Morning.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Monday-WorkingHours.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Tuesday-WorkingHours.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Wednesday-workingHours.pcap_ISCX.csv\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\n",
      "Uploaded 0 files\n",
      "\u001b[32mCommand ran in 7.632 seconds (init: 0.308, invoke: 7.323)\u001b[0m\n",
      "\u001b[0mCPU times: user 349 ms, sys: 104 ms, total: 453 ms\n",
      "Wall time: 8.95 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2017full --target-path \\\n",
    "    $dataset_2017_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Upload and register the uploaded datasets into AzureML workspace. In principle the above `az` commands should not be necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7fc5fc88e890>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2018train/data.csv\"], \"target_path\": \"ids2018train_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 1 files\n",
      "{\"message\": \"Uploading an estimated of 1 files\"}\n",
      "Target already exists. Skipping upload for ids2018train_1/data.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2018train_1/data.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"Dataset.File.from_files\", \"datastore_path\": [\"<azureml.data.datapath.DataPath object at 0x7fc5fc8a2a50>\"]}\n",
      "{\"message\": \"file_dataset.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2018train_1\", \"create_new_version\": false}}\n",
      "CPU times: user 5.62 s, sys: 1.04 s, total: 6.66 s\n",
      "Wall time: 1min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "    --dataset-path datasets --dataset-name ids2018train --dataset-version 1 \\\n",
    "    --dataset-type file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7fa5ee6b0850>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2018validate/data.csv\"], \"target_path\": \"ids2018validate_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 1 files\n",
      "{\"message\": \"Uploading an estimated of 1 files\"}\n",
      "Target already exists. Skipping upload for ids2018validate_1/data.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2018validate_1/data.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"Dataset.File.from_files\", \"datastore_path\": [\"<azureml.data.datapath.DataPath object at 0x7fa5ee6b8150>\"]}\n",
      "{\"message\": \"file_dataset.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2018validate_1\", \"create_new_version\": false}}\n",
      "CPU times: user 934 ms, sys: 150 ms, total: 1.08 s\n",
      "Wall time: 23.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "    --dataset-path datasets --dataset-name ids2018validate --dataset-version 1 \\\n",
    "    --dataset-type file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7fc7c82507d0>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2018test/data.csv\"], \"target_path\": \"ids2018test_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 1 files\n",
      "{\"message\": \"Uploading an estimated of 1 files\"}\n",
      "Target already exists. Skipping upload for ids2018test_1/data.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2018test_1/data.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"TabularDatasetFactory.from_delimited_files\", \"kwargs\": {\"path\": [\"<azureml.data.datapath.DataPath object at 0x7fc7c8257a50>\"], \"set_column_types\": {\"Flow Duration\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"TotLen Fwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"TotLen Bwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Fwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Bwd Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Bwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Flow Byts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Flow Pkts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Flow IAT Max\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Bwd IAT Min\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Bwd Header Len\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"RST Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"PSH Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"ECE Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Init Fwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Init Bwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Fwd Seg Size Min\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85d9610>\", \"Label\": \"<azureml.data.dataset_factory.DataType object at 0x7fc7c85dc410>\"}}}\n",
      "{\"message\": \"tabular.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2018test_1\", \"create_new_version\": false}}\n",
      "CPU times: user 3.62 s, sys: 734 ms, total: 4.35 s\n",
      "Wall time: 1min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "     --dataset-path datasets --dataset-name ids2018test --dataset-version 1 \\\n",
    "     --dataset-type tabular"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7f619328b890>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2017full/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Friday-WorkingHours-Morning.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Monday-WorkingHours.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Tuesday-WorkingHours.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Wednesday-workingHours.pcap_ISCX.csv\"], \"target_path\": \"ids2017full_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 8 files\n",
      "{\"message\": \"Uploading an estimated of 8 files\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Morning.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Morning.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Monday-WorkingHours.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Monday-WorkingHours.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Tuesday-WorkingHours.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Tuesday-WorkingHours.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Wednesday-workingHours.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Wednesday-workingHours.pcap_ISCX.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"TabularDatasetFactory.from_delimited_files\", \"kwargs\": {\"path\": [\"<azureml.data.datapath.DataPath object at 0x7f619327e450>\", \"<azureml.data.datapath.DataPath object at 0x7f619327e790>\", \"<azureml.data.datapath.DataPath object at 0x7f619327eb90>\", \"<azureml.data.datapath.DataPath object at 0x7f619327ec50>\", \"<azureml.data.datapath.DataPath object at 0x7f619327e9d0>\", \"<azureml.data.datapath.DataPath object at 0x7f61932284d0>\", \"<azureml.data.datapath.DataPath object at 0x7f6193228150>\", \"<azureml.data.datapath.DataPath object at 0x7f6193228b50>\"], \"set_column_types\": {\"Flow Duration\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"TotLen Fwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"TotLen Bwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Fwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Bwd Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Bwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Flow Byts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Flow Pkts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Flow IAT Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Bwd IAT Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Bwd Header Len\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"RST Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"PSH Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"ECE Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Init Fwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Init Bwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Fwd Seg Size Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193675910>\", \"Label\": \"<azureml.data.dataset_factory.DataType object at 0x7f6193610710>\"}}}\n",
      "{\"message\": \"tabular.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2017full_1\", \"create_new_version\": false}}\n",
      "CPU times: user 2.48 s, sys: 436 ms, total: 2.92 s\n",
      "Wall time: 51.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "     --dataset-path datasets --dataset-name ids2017full --dataset-version 1 \\\n",
    "     --dataset-type tabular"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "gather": {
     "logged": 1598531923519
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## HyperDrive Configuration\n",
    "\n",
    "The hyperparameter tuning is performed using a grid search on the [RandomForestClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html). The following hyperparameters are varied:\n",
    "-  the number of trees in the forest `n_estimators`, which affects the model generalization, however more trees will result in longer individual model training time,\n",
    "- the `criterion` affects the sensitivity of the model to the minority classes, which is important for imbalanced datasets, like the one used here,\n",
    "- the `max_depth` may result in overfitting if set too high, moreover it may considerably increase the size of the model (to e.g. several hundreds Mbytes).\n",
    "\n",
    "The ` BanditPolicy` termination policy is set such that it allows the grid search to explore all hyperparameter values, and the `max_concurrent_runs` is set to the number of grid search runs. The individual model runs are performed in parallel using all cores available on the compute instance by setting `n_jobs=-1`. The number of `max_concurrent_runs` is set to the number of the nodes in the compute cluster.\n",
    "\n",
    "See [nd00333/model/hyperdrive/train/run_config.py](nd00333/model/hyperdrive/train/run_config.py) for more details.\n",
    "\n",
    "The `norm_macro_recall` is used as the performance metrics. See [README.md](README.md) for the rationale."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perform the configuration of the HyperDrive run, including setting up the remote AML compute cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "gather": {
     "logged": 1598544893076
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found existing cluster starts-ws-136633\"}\n",
      "{\"message\": \"Found existing cluster starts-ws-136633\"}\n",
      "{\"message\": \"Found existing cluster starts-ws-136633\"}\n",
      "{\"message\": \"Found existing cluster starts-ws-136633\"}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 1, \"targetNodeCount\": 5, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 1, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Resizing\", \"allocationStateTransitionTime\": \"2021-01-30T19:19:00.161000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hd-train,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hd-train,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hd-train,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hd-train,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"run_config\": \"{\\n    \\\"script\\\": null,\\n    \\\"arguments\\\": [],\\n    \\\"target\\\": \\\"starts-ws-136633\\\",\\n    \\\"framework\\\": \\\"Python\\\",\\n    \\\"communicator\\\": \\\"None\\\",\\n    \\\"maxRunDurationSeconds\\\": null,\\n    \\\"nodeCount\\\": 1,\\n    \\\"environment\\\": {\\n        \\\"name\\\": \\\"hd-train\\\",\\n        \\\"version\\\": null,\\n        \\\"environmentVariables\\\": {\\n            \\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"\\n        },\\n        \\\"python\\\": {\\n            \\\"userManagedDependencies\\\": false,\\n            \\\"interpreterPath\\\": \\\"python\\\",\\n            \\\"condaDependenciesFile\\\": null,\\n            \\\"baseCondaEnvironment\\\": null,\\n            \\\"condaDependencies\\\": {\\n                \\\"name\\\": \\\"train\\\",\\n                \\\"channels\\\": [\\n                    \\\"conda-forge\\\"\\n                ],\\n                \\\"dependencies\\\": [\\n                    \\\"python=3.7\\\",\\n                    \\\"pip=20.1\\\",\\n                    \\\"numpy=1.18\\\",\\n                    \\\"scipy=1.4\\\",\\n                    \\\"joblib=0.14\\\",\\n                    \\\"scikit-learn=0.22\\\",\\n                    \\\"pandas=0.25\\\",\\n                    \\\"pyarrow=1.0.1\\\",\\n                    \\\"pyjwt=1.7\\\",\\n                    \\\"python-json-logger=2.0\\\",\\n                    {\\n                        \\\"pip\\\": [\\n                            \\\"azureml-sdk==1.21.0\\\",\\n                            \\\"azureml-defaults==1.21.0\\\",\\n                            \\\"azureml-interpret==1.21.0\\\",\\n                            \\\"azureml-opendatasets==1.21.0\\\",\\n                            \\\"azureml-train-automl==1.21.0\\\",\\n                            \\\"azureml-widgets==1.21.0\\\",\\n                            \\\"inference-schema==1.1.0\\\",\\n                            \\\"onnx==1.8.0\\\"\\n                        ]\\n                    }\\n                ]\\n            }\\n        },\\n        \\\"docker\\\": {\\n            \\\"enabled\\\": true,\\n            \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1\\\",\\n            \\\"baseDockerfile\\\": null,\\n            \\\"sharedVolumes\\\": true,\\n            \\\"shmSize\\\": \\\"2g\\\",\\n            \\\"arguments\\\": [],\\n            \\\"baseImageRegistry\\\": {\\n                \\\"address\\\": null,\\n                \\\"username\\\": null,\\n                \\\"password\\\": null,\\n                \\\"registryIdentity\\\": null\\n            },\\n            \\\"platform\\\": {\\n                \\\"os\\\": \\\"Linux\\\",\\n                \\\"architecture\\\": \\\"amd64\\\"\\n            }\\n        },\\n        \\\"spark\\\": {\\n            \\\"repositories\\\": [],\\n            \\\"packages\\\": [],\\n            \\\"precachePackages\\\": true\\n        },\\n        \\\"databricks\\\": {\\n            \\\"mavenLibraries\\\": [],\\n            \\\"pypiLibraries\\\": [],\\n            \\\"rcranLibraries\\\": [],\\n            \\\"jarLibraries\\\": [],\\n            \\\"eggLibraries\\\": []\\n        },\\n        \\\"r\\\": null,\\n        \\\"inferencingStackVersion\\\": null\\n    },\\n    \\\"history\\\": {\\n        \\\"outputCollection\\\": true,\\n        \\\"snapshotProject\\\": true,\\n        \\\"directoriesToWatch\\\": [\\n            \\\"logs\\\"\\n        ]\\n    },\\n    \\\"spark\\\": {\\n        \\\"configuration\\\": {\\n            \\\"spark.app.name\\\": \\\"Azure ML Experiment\\\",\\n            \\\"spark.yarn.maxAppAttempts\\\": 1\\n        }\\n    },\\n    \\\"hdi\\\": {\\n        \\\"yarnDeployMode\\\": \\\"cluster\\\"\\n    },\\n    \\\"tensorflow\\\": {\\n        \\\"workerCount\\\": 1,\\n        \\\"parameterServerCount\\\": 1\\n    },\\n    \\\"mpi\\\": {\\n        \\\"processCountPerNode\\\": 1,\\n        \\\"nodeCount\\\": 1\\n    },\\n    \\\"paralleltask\\\": {\\n        \\\"maxRetriesPerWorker\\\": 0,\\n        \\\"workerCountPerNode\\\": 1,\\n        \\\"terminalExitCodes\\\": null\\n    },\\n    \\\"dataReferences\\\": {},\\n    \\\"data\\\": {},\\n    \\\"outputData\\\": {},\\n    \\\"sourceDirectoryDataStore\\\": null,\\n    \\\"amlcompute\\\": {\\n        \\\"vmSize\\\": null,\\n        \\\"vmPriority\\\": null,\\n        \\\"retainCluster\\\": false,\\n        \\\"name\\\": null,\\n        \\\"clusterMaxNodeCount\\\": null\\n    },\\n    \\\"command\\\": \\\"\\\"\\n}\"}\n",
      "{\"message\": \"main\", \"run_config\": \"{\\n    \\\"script\\\": null,\\n    \\\"arguments\\\": [],\\n    \\\"target\\\": \\\"starts-ws-136633\\\",\\n    \\\"framework\\\": \\\"Python\\\",\\n    \\\"communicator\\\": \\\"None\\\",\\n    \\\"maxRunDurationSeconds\\\": null,\\n    \\\"nodeCount\\\": 1,\\n    \\\"environment\\\": {\\n        \\\"name\\\": \\\"hd-train\\\",\\n        \\\"version\\\": null,\\n        \\\"environmentVariables\\\": {\\n            \\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"\\n        },\\n        \\\"python\\\": {\\n            \\\"userManagedDependencies\\\": false,\\n            \\\"interpreterPath\\\": \\\"python\\\",\\n            \\\"condaDependenciesFile\\\": null,\\n            \\\"baseCondaEnvironment\\\": null,\\n            \\\"condaDependencies\\\": {\\n                \\\"name\\\": \\\"train\\\",\\n                \\\"channels\\\": [\\n                    \\\"conda-forge\\\"\\n                ],\\n                \\\"dependencies\\\": [\\n                    \\\"python=3.7\\\",\\n                    \\\"pip=20.1\\\",\\n                    \\\"numpy=1.18\\\",\\n                    \\\"scipy=1.4\\\",\\n                    \\\"joblib=0.14\\\",\\n                    \\\"scikit-learn=0.22\\\",\\n                    \\\"pandas=0.25\\\",\\n                    \\\"pyarrow=1.0.1\\\",\\n                    \\\"pyjwt=1.7\\\",\\n                    \\\"python-json-logger=2.0\\\",\\n                    {\\n                        \\\"pip\\\": [\\n                            \\\"azureml-sdk==1.21.0\\\",\\n                            \\\"azureml-defaults==1.21.0\\\",\\n                            \\\"azureml-interpret==1.21.0\\\",\\n                            \\\"azureml-opendatasets==1.21.0\\\",\\n                            \\\"azureml-train-automl==1.21.0\\\",\\n                            \\\"azureml-widgets==1.21.0\\\",\\n                            \\\"inference-schema==1.1.0\\\",\\n                            \\\"onnx==1.8.0\\\"\\n                        ]\\n                    }\\n                ]\\n            }\\n        },\\n        \\\"docker\\\": {\\n            \\\"enabled\\\": true,\\n            \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1\\\",\\n            \\\"baseDockerfile\\\": null,\\n            \\\"sharedVolumes\\\": true,\\n            \\\"shmSize\\\": \\\"2g\\\",\\n            \\\"arguments\\\": [],\\n            \\\"baseImageRegistry\\\": {\\n                \\\"address\\\": null,\\n                \\\"username\\\": null,\\n                \\\"password\\\": null,\\n                \\\"registryIdentity\\\": null\\n            },\\n            \\\"platform\\\": {\\n                \\\"os\\\": \\\"Linux\\\",\\n                \\\"architecture\\\": \\\"amd64\\\"\\n            }\\n        },\\n        \\\"spark\\\": {\\n            \\\"repositories\\\": [],\\n            \\\"packages\\\": [],\\n            \\\"precachePackages\\\": true\\n        },\\n        \\\"databricks\\\": {\\n            \\\"mavenLibraries\\\": [],\\n            \\\"pypiLibraries\\\": [],\\n            \\\"rcranLibraries\\\": [],\\n            \\\"jarLibraries\\\": [],\\n            \\\"eggLibraries\\\": []\\n        },\\n        \\\"r\\\": null,\\n        \\\"inferencingStackVersion\\\": null\\n    },\\n    \\\"history\\\": {\\n        \\\"outputCollection\\\": true,\\n        \\\"snapshotProject\\\": true,\\n        \\\"directoriesToWatch\\\": [\\n            \\\"logs\\\"\\n        ]\\n    },\\n    \\\"spark\\\": {\\n        \\\"configuration\\\": {\\n            \\\"spark.app.name\\\": \\\"Azure ML Experiment\\\",\\n            \\\"spark.yarn.maxAppAttempts\\\": 1\\n        }\\n    },\\n    \\\"hdi\\\": {\\n        \\\"yarnDeployMode\\\": \\\"cluster\\\"\\n    },\\n    \\\"tensorflow\\\": {\\n        \\\"workerCount\\\": 1,\\n        \\\"parameterServerCount\\\": 1\\n    },\\n    \\\"mpi\\\": {\\n        \\\"processCountPerNode\\\": 1,\\n        \\\"nodeCount\\\": 1\\n    },\\n    \\\"paralleltask\\\": {\\n        \\\"maxRetriesPerWorker\\\": 0,\\n        \\\"workerCountPerNode\\\": 1,\\n        \\\"terminalExitCodes\\\": null\\n    },\\n    \\\"dataReferences\\\": {},\\n    \\\"data\\\": {},\\n    \\\"outputData\\\": {},\\n    \\\"sourceDirectoryDataStore\\\": null,\\n    \\\"amlcompute\\\": {\\n        \\\"vmSize\\\": null,\\n        \\\"vmPriority\\\": null,\\n        \\\"retainCluster\\\": false,\\n        \\\"name\\\": null,\\n        \\\"clusterMaxNodeCount\\\": null\\n    },\\n    \\\"command\\\": \\\"\\\"\\n}\"}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"main\", \"run_config\": \"{\\n    \\\"script\\\": null,\\n    \\\"arguments\\\": [],\\n    \\\"target\\\": \\\"starts-ws-136633\\\",\\n    \\\"framework\\\": \\\"Python\\\",\\n    \\\"communicator\\\": \\\"None\\\",\\n    \\\"maxRunDurationSeconds\\\": null,\\n    \\\"nodeCount\\\": 1,\\n    \\\"environment\\\": {\\n        \\\"name\\\": \\\"hd-train\\\",\\n        \\\"version\\\": null,\\n        \\\"environmentVariables\\\": {\\n            \\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"\\n        },\\n        \\\"python\\\": {\\n            \\\"userManagedDependencies\\\": false,\\n            \\\"interpreterPath\\\": \\\"python\\\",\\n            \\\"condaDependenciesFile\\\": null,\\n            \\\"baseCondaEnvironment\\\": null,\\n            \\\"condaDependencies\\\": {\\n                \\\"name\\\": \\\"train\\\",\\n                \\\"channels\\\": [\\n                    \\\"conda-forge\\\"\\n                ],\\n                \\\"dependencies\\\": [\\n                    \\\"python=3.7\\\",\\n                    \\\"pip=20.1\\\",\\n                    \\\"numpy=1.18\\\",\\n                    \\\"scipy=1.4\\\",\\n                    \\\"joblib=0.14\\\",\\n                    \\\"scikit-learn=0.22\\\",\\n                    \\\"pandas=0.25\\\",\\n                    \\\"pyarrow=1.0.1\\\",\\n                    \\\"pyjwt=1.7\\\",\\n                    \\\"python-json-logger=2.0\\\",\\n                    {\\n                        \\\"pip\\\": [\\n                            \\\"azureml-sdk==1.21.0\\\",\\n                            \\\"azureml-defaults==1.21.0\\\",\\n                            \\\"azureml-interpret==1.21.0\\\",\\n                            \\\"azureml-opendatasets==1.21.0\\\",\\n                            \\\"azureml-train-automl==1.21.0\\\",\\n                            \\\"azureml-widgets==1.21.0\\\",\\n                            \\\"inference-schema==1.1.0\\\",\\n                            \\\"onnx==1.8.0\\\"\\n                        ]\\n                    }\\n                ]\\n            }\\n        },\\n        \\\"docker\\\": {\\n            \\\"enabled\\\": true,\\n            \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1\\\",\\n            \\\"baseDockerfile\\\": null,\\n            \\\"sharedVolumes\\\": true,\\n            \\\"shmSize\\\": \\\"2g\\\",\\n            \\\"arguments\\\": [],\\n            \\\"baseImageRegistry\\\": {\\n                \\\"address\\\": null,\\n                \\\"username\\\": null,\\n                \\\"password\\\": null,\\n                \\\"registryIdentity\\\": null\\n            },\\n            \\\"platform\\\": {\\n                \\\"os\\\": \\\"Linux\\\",\\n                \\\"architecture\\\": \\\"amd64\\\"\\n            }\\n        },\\n        \\\"spark\\\": {\\n            \\\"repositories\\\": [],\\n            \\\"packages\\\": [],\\n            \\\"precachePackages\\\": true\\n        },\\n        \\\"databricks\\\": {\\n            \\\"mavenLibraries\\\": [],\\n            \\\"pypiLibraries\\\": [],\\n            \\\"rcranLibraries\\\": [],\\n            \\\"jarLibraries\\\": [],\\n            \\\"eggLibraries\\\": []\\n        },\\n        \\\"r\\\": null,\\n        \\\"inferencingStackVersion\\\": null\\n    },\\n    \\\"history\\\": {\\n        \\\"outputCollection\\\": true,\\n        \\\"snapshotProject\\\": true,\\n        \\\"directoriesToWatch\\\": [\\n            \\\"logs\\\"\\n        ]\\n    },\\n    \\\"spark\\\": {\\n        \\\"configuration\\\": {\\n            \\\"spark.app.name\\\": \\\"Azure ML Experiment\\\",\\n            \\\"spark.yarn.maxAppAttempts\\\": 1\\n        }\\n    },\\n    \\\"hdi\\\": {\\n        \\\"yarnDeployMode\\\": \\\"cluster\\\"\\n    },\\n    \\\"tensorflow\\\": {\\n        \\\"workerCount\\\": 1,\\n        \\\"parameterServerCount\\\": 1\\n    },\\n    \\\"mpi\\\": {\\n        \\\"processCountPerNode\\\": 1,\\n        \\\"nodeCount\\\": 1\\n    },\\n    \\\"paralleltask\\\": {\\n        \\\"maxRetriesPerWorker\\\": 0,\\n        \\\"workerCountPerNode\\\": 1,\\n        \\\"terminalExitCodes\\\": null\\n    },\\n    \\\"dataReferences\\\": {},\\n    \\\"data\\\": {},\\n    \\\"outputData\\\": {},\\n    \\\"sourceDirectoryDataStore\\\": null,\\n    \\\"amlcompute\\\": {\\n        \\\"vmSize\\\": null,\\n        \\\"vmPriority\\\": null,\\n        \\\"retainCluster\\\": false,\\n        \\\"name\\\": null,\\n        \\\"clusterMaxNodeCount\\\": null\\n    },\\n    \\\"command\\\": \\\"\\\"\\n}\"}\n",
      "{\"message\": \"main\", \"run_config\": \"{\\n    \\\"script\\\": null,\\n    \\\"arguments\\\": [],\\n    \\\"target\\\": \\\"starts-ws-136633\\\",\\n    \\\"framework\\\": \\\"Python\\\",\\n    \\\"communicator\\\": \\\"None\\\",\\n    \\\"maxRunDurationSeconds\\\": null,\\n    \\\"nodeCount\\\": 1,\\n    \\\"environment\\\": {\\n        \\\"name\\\": \\\"hd-train\\\",\\n        \\\"version\\\": null,\\n        \\\"environmentVariables\\\": {\\n            \\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"\\n        },\\n        \\\"python\\\": {\\n            \\\"userManagedDependencies\\\": false,\\n            \\\"interpreterPath\\\": \\\"python\\\",\\n            \\\"condaDependenciesFile\\\": null,\\n            \\\"baseCondaEnvironment\\\": null,\\n            \\\"condaDependencies\\\": {\\n                \\\"name\\\": \\\"train\\\",\\n                \\\"channels\\\": [\\n                    \\\"conda-forge\\\"\\n                ],\\n                \\\"dependencies\\\": [\\n                    \\\"python=3.7\\\",\\n                    \\\"pip=20.1\\\",\\n                    \\\"numpy=1.18\\\",\\n                    \\\"scipy=1.4\\\",\\n                    \\\"joblib=0.14\\\",\\n                    \\\"scikit-learn=0.22\\\",\\n                    \\\"pandas=0.25\\\",\\n                    \\\"pyarrow=1.0.1\\\",\\n                    \\\"pyjwt=1.7\\\",\\n                    \\\"python-json-logger=2.0\\\",\\n                    {\\n                        \\\"pip\\\": [\\n                            \\\"azureml-sdk==1.21.0\\\",\\n                            \\\"azureml-defaults==1.21.0\\\",\\n                            \\\"azureml-interpret==1.21.0\\\",\\n                            \\\"azureml-opendatasets==1.21.0\\\",\\n                            \\\"azureml-train-automl==1.21.0\\\",\\n                            \\\"azureml-widgets==1.21.0\\\",\\n                            \\\"inference-schema==1.1.0\\\",\\n                            \\\"onnx==1.8.0\\\"\\n                        ]\\n                    }\\n                ]\\n            }\\n        },\\n        \\\"docker\\\": {\\n            \\\"enabled\\\": true,\\n            \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1\\\",\\n            \\\"baseDockerfile\\\": null,\\n            \\\"sharedVolumes\\\": true,\\n            \\\"shmSize\\\": \\\"2g\\\",\\n            \\\"arguments\\\": [],\\n            \\\"baseImageRegistry\\\": {\\n                \\\"address\\\": null,\\n                \\\"username\\\": null,\\n                \\\"password\\\": null,\\n                \\\"registryIdentity\\\": null\\n            },\\n            \\\"platform\\\": {\\n                \\\"os\\\": \\\"Linux\\\",\\n                \\\"architecture\\\": \\\"amd64\\\"\\n            }\\n        },\\n        \\\"spark\\\": {\\n            \\\"repositories\\\": [],\\n            \\\"packages\\\": [],\\n            \\\"precachePackages\\\": true\\n        },\\n        \\\"databricks\\\": {\\n            \\\"mavenLibraries\\\": [],\\n            \\\"pypiLibraries\\\": [],\\n            \\\"rcranLibraries\\\": [],\\n            \\\"jarLibraries\\\": [],\\n            \\\"eggLibraries\\\": []\\n        },\\n        \\\"r\\\": null,\\n        \\\"inferencingStackVersion\\\": null\\n    },\\n    \\\"history\\\": {\\n        \\\"outputCollection\\\": true,\\n        \\\"snapshotProject\\\": true,\\n        \\\"directoriesToWatch\\\": [\\n            \\\"logs\\\"\\n        ]\\n    },\\n    \\\"spark\\\": {\\n        \\\"configuration\\\": {\\n            \\\"spark.app.name\\\": \\\"Azure ML Experiment\\\",\\n            \\\"spark.yarn.maxAppAttempts\\\": 1\\n        }\\n    },\\n    \\\"hdi\\\": {\\n        \\\"yarnDeployMode\\\": \\\"cluster\\\"\\n    },\\n    \\\"tensorflow\\\": {\\n        \\\"workerCount\\\": 1,\\n        \\\"parameterServerCount\\\": 1\\n    },\\n    \\\"mpi\\\": {\\n        \\\"processCountPerNode\\\": 1,\\n        \\\"nodeCount\\\": 1\\n    },\\n    \\\"paralleltask\\\": {\\n        \\\"maxRetriesPerWorker\\\": 0,\\n        \\\"workerCountPerNode\\\": 1,\\n        \\\"terminalExitCodes\\\": null\\n    },\\n    \\\"dataReferences\\\": {},\\n    \\\"data\\\": {},\\n    \\\"outputData\\\": {},\\n    \\\"sourceDirectoryDataStore\\\": null,\\n    \\\"amlcompute\\\": {\\n        \\\"vmSize\\\": null,\\n        \\\"vmPriority\\\": null,\\n        \\\"retainCluster\\\": false,\\n        \\\"name\\\": null,\\n        \\\"clusterMaxNodeCount\\\": null\\n    },\\n    \\\"command\\\": \\\"\\\"\\n}\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"If 'run_config' is specified, the following parameters will be ignored: 'compute_target', 'environment', 'distributed_job_config', and 'max_run_duration_seconds'.\"}\n",
      "{\"message\": \"If 'run_config' is specified, the following parameters will be ignored: 'compute_target', 'environment', 'distributed_job_config', and 'max_run_duration_seconds'.\"}\n",
      "{\"message\": \"If 'run_config' is specified, the following parameters will be ignored: 'compute_target', 'environment', 'distributed_job_config', and 'max_run_duration_seconds'.\"}\n",
      "{\"message\": \"If 'run_config' is specified, the following parameters will be ignored: 'compute_target', 'environment', 'distributed_job_config', and 'max_run_duration_seconds'.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 994 ms, sys: 72.1 ms, total: 1.07 s\n",
      "Wall time: 7.27 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hyperdrive_run_config = hd_train_config.main(dataset_train_name=dataset_train_name,\n",
    "                                             dataset_validate_name=dataset_validate_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Submit the HyperDrive configuration to the compute cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "gather": {
     "logged": 1598544897941
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"aa1dacbd-43b9-4cf1-be80-2f2a011a93e3\", \"dataset_name\": \"ids2018train_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"aa1dacbd-43b9-4cf1-be80-2f2a011a93e3\", \"dataset_name\": \"ids2018train_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"aa1dacbd-43b9-4cf1-be80-2f2a011a93e3\", \"dataset_name\": \"ids2018train_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"aa1dacbd-43b9-4cf1-be80-2f2a011a93e3\", \"dataset_name\": \"ids2018train_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"5412201c-2a44-4b6d-8c05-74d04d884006\", \"dataset_name\": \"ids2018validate_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"5412201c-2a44-4b6d-8c05-74d04d884006\", \"dataset_name\": \"ids2018validate_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"5412201c-2a44-4b6d-8c05-74d04d884006\", \"dataset_name\": \"ids2018validate_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"HyperDriveSubmit\", \"properties\": {\"core_sdk_version\": \"1.21.0\", \"input_type\": \"FileDataset\", \"dataset_id\": \"5412201c-2a44-4b6d-8c05-74d04d884006\", \"dataset_name\": \"ids2018validate_1\", \"dataset_version\": \"dataset.dataset.version\", \"consumption_mode\": \"mount\", \"compute\": \"starts-ws-136633\", \"subscription_id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\", \"resource_group_name\": \"aml-quickstarts-136633\", \"workspace_name\": \"quick-starts-ws-136633\", \"location\": \"southcentralus\"}}\n",
      "{\"message\": \"Connection pool is full, discarding connection: mlstrg136633.blob.core.windows.net\"}\n",
      "{\"message\": \"Connection pool is full, discarding connection: mlstrg136633.blob.core.windows.net\"}\n",
      "{\"message\": \"Connection pool is full, discarding connection: mlstrg136633.blob.core.windows.net\"}\n",
      "{\"message\": \"Connection pool is full, discarding connection: mlstrg136633.blob.core.windows.net\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.25 s, sys: 303 ms, total: 1.55 s\n",
      "Wall time: 6.64 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "hyperdrive_run = experiment.submit(config=hyperdrive_run_config, show_output=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RunId: HD_67ab13fd-0409-4b2d-807f-a943d6e47e67\n",
      "Web View: https://ml.azure.com/experiments/jupyter-2021-01-29-09-01/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-136633/workspaces/quick-starts-ws-136633\n",
      "\n",
      "Streaming azureml-logs/hyperdrive.txt\n",
      "=====================================\n",
      "\n",
      "\"<START>[2021-01-30T19:19:35.655157][API][INFO]Experiment created<END>\\n\"\"<START>[2021-01-30T19:19:36.257793][GENERATOR][INFO]Trying to sample '5' jobs from the hyperparameter space<END>\\n\"\"<START>[2021-01-30T19:19:36.416485][GENERATOR][INFO]Successfully sampled '5' jobs, they will soon be submitted to the execution target.<END>\\n\"<START>[2021-01-30T19:19:36.8494663Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.<END>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"25980287-21bd-4c09-a801-3f667d185d95 - CacheDriver:Returning token refreshed after expiry.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Execution Summary\n",
      "=================\n",
      "RunId: HD_67ab13fd-0409-4b2d-807f-a943d6e47e67\n",
      "Web View: https://ml.azure.com/experiments/jupyter-2021-01-29-09-01/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-136633/workspaces/quick-starts-ws-136633\n",
      "\n",
      "CPU times: user 6.37 s, sys: 655 ms, total: 7.02 s\n",
      "Wall time: 47min 53s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'runId': 'HD_67ab13fd-0409-4b2d-807f-a943d6e47e67',\n",
       " 'target': 'starts-ws-136633',\n",
       " 'status': 'Completed',\n",
       " 'startTimeUtc': '2021-01-30T19:19:35.403137Z',\n",
       " 'endTimeUtc': '2021-01-30T20:06:45.919143Z',\n",
       " 'properties': {'primary_metric_config': '{\"name\": \"norm_macro_recall\", \"goal\": \"maximize\"}',\n",
       "  'resume_from': 'null',\n",
       "  'runTemplate': 'HyperDrive',\n",
       "  'azureml.runsource': 'hyperdrive',\n",
       "  'platform': 'AML',\n",
       "  'ContentSnapshotId': '6578d9a0-cd2c-45d5-95c1-ff450166ebd0',\n",
       "  'score': '0.858415258389476',\n",
       "  'best_child_run_id': 'HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9',\n",
       "  'best_metric_status': 'Succeeded'},\n",
       " 'inputDatasets': [],\n",
       " 'outputDatasets': [],\n",
       " 'logFiles': {'azureml-logs/hyperdrive.txt': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_67ab13fd-0409-4b2d-807f-a943d6e47e67/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=EZ4AUGwDVSQvPIg3OmROVq86YjaHAaF9enFZbgAiNpo%3D&st=2021-01-30T19%3A57%3A28Z&se=2021-01-31T04%3A07%3A28Z&sp=r'},\n",
       " 'submittedBy': 'ODL_User 136633'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "hyperdrive_run.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(hyperdrive_run.get_status() == \"Completed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "gather": {
     "logged": 1598544898497
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Run Details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The widget below shows the first few best runs of the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "gather": {
     "logged": 1598546648408
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8341968565664378a7a1e2c6faff76ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/jupyter-2021-01-29-09-01/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-136633/workspaces/quick-starts-ws-136633\", \"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"run_properties\": {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"created_utc\": \"2021-01-30T19:19:35.336457Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"norm_macro_recall\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"ContentSnapshotId\": \"6578d9a0-cd2c-45d5-95c1-ff450166ebd0\", \"score\": \"0.858415258389476\", \"best_child_run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9\", \"best_metric_status\": \"Succeeded\"}, \"tags\": {\"_aml_system_max_concurrent_jobs\": \"5\", \"max_concurrent_jobs\": \"5\", \"_aml_system_max_total_jobs\": \"12\", \"max_total_jobs\": \"12\", \"_aml_system_max_duration_minutes\": \"10080\", \"max_duration_minutes\": \"10080\", \"_aml_system_policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 1, \\\"delay_evaluation\\\": 0, \\\"slack_amount\\\": 1.0}}\", \"policy_config\": \"{\\\"name\\\": \\\"BANDIT\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 1, \\\"delay_evaluation\\\": 0, \\\"slack_amount\\\": 1.0}}\", \"_aml_system_generator_config\": \"{\\\"name\\\": \\\"GRID\\\", \\\"parameter_space\\\": {\\\"--hyperparameter-n_estimators\\\": [\\\"choice\\\", [[15, 20, 25]]], \\\"--hyperparameter-criterion\\\": [\\\"choice\\\", [[\\\"gini\\\", \\\"entropy\\\"]]], \\\"--hyperparameter-max_depth\\\": [\\\"choice\\\", [[10, 15]]]}}\", \"generator_config\": \"{\\\"name\\\": \\\"GRID\\\", \\\"parameter_space\\\": {\\\"--hyperparameter-n_estimators\\\": [\\\"choice\\\", [[15, 20, 25]]], \\\"--hyperparameter-criterion\\\": [\\\"choice\\\", [[\\\"gini\\\", \\\"entropy\\\"]]], \\\"--hyperparameter-max_depth\\\": [\\\"choice\\\", [[10, 15]]]}}\", \"_aml_system_primary_metric_config\": \"{\\\"name\\\": \\\"norm_macro_recall\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"primary_metric_config\": \"{\\\"name\\\": \\\"norm_macro_recall\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"_aml_system_platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/jupyter-2021-01-29-09-01\\\", \\\"SubscriptionId\\\": \\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-136633\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-136633\\\", \\\"ExperimentName\\\": \\\"jupyter-2021-01-29-09-01\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [\\\"--dataset-train-path\\\", \\\"DatasetConsumptionConfig:train\\\", \\\"--dataset-validate-path\\\", \\\"DatasetConsumptionConfig:validate\\\", \\\"--hyperparameter-n_jobs\\\", -1, \\\"--hyperparameter-random_state\\\", 0], \\\"target\\\": \\\"starts-ws-136633\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": \\\"hd-train\\\", \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"train\\\", \\\"channels\\\": [\\\"conda-forge\\\"], \\\"dependencies\\\": [\\\"python=3.7\\\", \\\"pip=20.1\\\", \\\"numpy=1.18\\\", \\\"scipy=1.4\\\", \\\"joblib=0.14\\\", \\\"scikit-learn=0.22\\\", \\\"pandas=0.25\\\", \\\"pyarrow=1.0.1\\\", \\\"pyjwt=1.7\\\", \\\"python-json-logger=2.0\\\", {\\\"pip\\\": [\\\"azureml-sdk==1.21.0\\\", \\\"azureml-defaults==1.21.0\\\", \\\"azureml-interpret==1.21.0\\\", \\\"azureml-opendatasets==1.21.0\\\", \\\"azureml-train-automl==1.21.0\\\", \\\"azureml-widgets==1.21.0\\\", \\\"inference-schema==1.1.0\\\", \\\"onnx==1.8.0\\\"]}]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": true}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {\\\"train\\\": {\\\"dataLocation\\\": {\\\"dataset\\\": {\\\"id\\\": \\\"aa1dacbd-43b9-4cf1-be80-2f2a011a93e3\\\", \\\"name\\\": \\\"ids2018train_1\\\", \\\"version\\\": 1}, \\\"dataPath\\\": null}, \\\"createOutputDirectories\\\": false, \\\"mechanism\\\": \\\"mount\\\", \\\"environmentVariableName\\\": \\\"train\\\", \\\"pathOnCompute\\\": null, \\\"overwrite\\\": false}, \\\"validate\\\": {\\\"dataLocation\\\": {\\\"dataset\\\": {\\\"id\\\": \\\"5412201c-2a44-4b6d-8c05-74d04d884006\\\", \\\"name\\\": \\\"ids2018validate_1\\\", \\\"version\\\": 1}, \\\"dataPath\\\": null}, \\\"createOutputDirectories\\\": false, \\\"mechanism\\\": \\\"mount\\\", \\\"environmentVariableName\\\": \\\"validate\\\", \\\"pathOnCompute\\\": null, \\\"overwrite\\\": false}}, \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": null}, \\\"command\\\": \\\"\\\"}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"6578d9a0-cd2c-45d5-95c1-ff450166ebd0\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"6c7f4ba9-4b01-42ed-bde8-bb4f8caadafc\\\", \\\"amlClientSessionId\\\": \\\"5acdef55-c29d-4254-b576-59ae3df22a24\\\", \\\"subscriptionId\\\": \\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\", \\\"estimator\\\": \\\"NoneType\\\", \\\"samplingMethod\\\": \\\"GRID\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 12, \\\"maxConcurrentRuns\\\": 5, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": null}}}\", \"platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://southcentralus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/jupyter-2021-01-29-09-01\\\", \\\"SubscriptionId\\\": \\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\", \\\"ResourceGroupName\\\": \\\"aml-quickstarts-136633\\\", \\\"WorkspaceName\\\": \\\"quick-starts-ws-136633\\\", \\\"ExperimentName\\\": \\\"jupyter-2021-01-29-09-01\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train.py\\\", \\\"arguments\\\": [\\\"--dataset-train-path\\\", \\\"DatasetConsumptionConfig:train\\\", \\\"--dataset-validate-path\\\", \\\"DatasetConsumptionConfig:validate\\\", \\\"--hyperparameter-n_jobs\\\", -1, \\\"--hyperparameter-random_state\\\", 0], \\\"target\\\": \\\"starts-ws-136633\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": \\\"hd-train\\\", \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"train\\\", \\\"channels\\\": [\\\"conda-forge\\\"], \\\"dependencies\\\": [\\\"python=3.7\\\", \\\"pip=20.1\\\", \\\"numpy=1.18\\\", \\\"scipy=1.4\\\", \\\"joblib=0.14\\\", \\\"scikit-learn=0.22\\\", \\\"pandas=0.25\\\", \\\"pyarrow=1.0.1\\\", \\\"pyjwt=1.7\\\", \\\"python-json-logger=2.0\\\", {\\\"pip\\\": [\\\"azureml-sdk==1.21.0\\\", \\\"azureml-defaults==1.21.0\\\", \\\"azureml-interpret==1.21.0\\\", \\\"azureml-opendatasets==1.21.0\\\", \\\"azureml-train-automl==1.21.0\\\", \\\"azureml-widgets==1.21.0\\\", \\\"inference-schema==1.1.0\\\", \\\"onnx==1.8.0\\\"]}]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20200821.v1\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null, \\\"registryIdentity\\\": null}, \\\"platform\\\": {\\\"os\\\": \\\"Linux\\\", \\\"architecture\\\": \\\"amd64\\\"}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": true}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"r\\\": null, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1, \\\"nodeCount\\\": 1}, \\\"paralleltask\\\": {\\\"maxRetriesPerWorker\\\": 0, \\\"workerCountPerNode\\\": 1, \\\"terminalExitCodes\\\": null}, \\\"dataReferences\\\": {}, \\\"data\\\": {\\\"train\\\": {\\\"dataLocation\\\": {\\\"dataset\\\": {\\\"id\\\": \\\"aa1dacbd-43b9-4cf1-be80-2f2a011a93e3\\\", \\\"name\\\": \\\"ids2018train_1\\\", \\\"version\\\": 1}, \\\"dataPath\\\": null}, \\\"createOutputDirectories\\\": false, \\\"mechanism\\\": \\\"mount\\\", \\\"environmentVariableName\\\": \\\"train\\\", \\\"pathOnCompute\\\": null, \\\"overwrite\\\": false}, \\\"validate\\\": {\\\"dataLocation\\\": {\\\"dataset\\\": {\\\"id\\\": \\\"5412201c-2a44-4b6d-8c05-74d04d884006\\\", \\\"name\\\": \\\"ids2018validate_1\\\", \\\"version\\\": 1}, \\\"dataPath\\\": null}, \\\"createOutputDirectories\\\": false, \\\"mechanism\\\": \\\"mount\\\", \\\"environmentVariableName\\\": \\\"validate\\\", \\\"pathOnCompute\\\": null, \\\"overwrite\\\": false}}, \\\"outputData\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": null}, \\\"command\\\": \\\"\\\"}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"6578d9a0-cd2c-45d5-95c1-ff450166ebd0\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"[Scrubbed]\\\", \\\"amlClientFunction\\\": \\\"[Scrubbed]\\\", \\\"tenantId\\\": \\\"660b3398-b80e-49d2-bc5b-ac1dc93b5254\\\", \\\"amlClientRequestId\\\": \\\"6c7f4ba9-4b01-42ed-bde8-bb4f8caadafc\\\", \\\"amlClientSessionId\\\": \\\"5acdef55-c29d-4254-b576-59ae3df22a24\\\", \\\"subscriptionId\\\": \\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\", \\\"estimator\\\": \\\"NoneType\\\", \\\"samplingMethod\\\": \\\"GRID\\\", \\\"terminationPolicy\\\": \\\"Bandit\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 12, \\\"maxConcurrentRuns\\\": 5, \\\"maxDurationMinutes\\\": 10080, \\\"vmSize\\\": null}}}\", \"_aml_system_resume_child_runs\": \"null\", \"resume_child_runs\": \"null\", \"_aml_system_all_jobs_generated\": \"true\", \"all_jobs_generated\": \"true\", \"_aml_system_cancellation_requested\": \"false\", \"cancellation_requested\": \"false\", \"_aml_system_progress_metadata_evaluation_timestamp\": \"\\\"2021-01-30T19:19:36.047230\\\"\", \"progress_metadata_evaluation_timestamp\": \"\\\"2021-01-30T19:19:36.047230\\\"\", \"_aml_system_progress_metadata_digest\": \"\\\"c8b6bdbca113082b908fba7ee72c063c53d5182a0ebaa2f572e63ccddffd7697\\\"\", \"progress_metadata_digest\": \"\\\"c8b6bdbca113082b908fba7ee72c063c53d5182a0ebaa2f572e63ccddffd7697\\\"\", \"_aml_system_progress_metadata_active_timestamp\": \"\\\"2021-01-30T19:19:36.047230\\\"\", \"progress_metadata_active_timestamp\": \"\\\"2021-01-30T19:19:36.047230\\\"\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"_aml_system_environment_preparation_status\": \"PREPARED\", \"environment_preparation_status\": \"PREPARED\", \"_aml_system_prepare_run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_preparation\", \"prepare_run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_preparation\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 10}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"gini\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9\": \"{\\\"--hyperparameter-n_estimators\\\": 15, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10\": \"{\\\"--hyperparameter-n_estimators\\\": 20, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"_aml_system_HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11\": \"{\\\"--hyperparameter-n_estimators\\\": 25, \\\"--hyperparameter-criterion\\\": \\\"entropy\\\", \\\"--hyperparameter-max_depth\\\": 15}\", \"_aml_system_final_best_metric_update_retry_count\": \"1\", \"final_best_metric_update_retry_count\": \"1\"}, \"end_time_utc\": \"2021-01-30T20:06:45.919143Z\", \"status\": \"Completed\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_67ab13fd-0409-4b2d-807f-a943d6e47e67/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=35BCQuFWCT6RE6gAQ17xGWwwwXoRCinj%2B%2FC5T90ZiNw%3D&st=2021-01-30T21%3A58%3A22Z&se=2021-01-31T06%3A08%3A22Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:47:10\", \"run_number\": \"1\", \"run_queued_details\": {\"status\": \"Completed\", \"details\": null}, \"hyper_parameters\": {\"--hyperparameter-n_estimators\": [\"choice\", [[15, 20, 25]]], \"--hyperparameter-criterion\": [\"choice\", [[\"gini\", \"entropy\"]]], \"--hyperparameter-max_depth\": [\"choice\", [[10, 15]]]}}, \"child_runs\": [{\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2\", \"run_number\": 3, \"metric\": 0.8064898, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:30:06.267768Z\", \"end_time\": \"2021-01-30T19:34:20.584979Z\", \"created_time\": \"2021-01-30T19:29:16.745702Z\", \"created_time_dt\": \"2021-01-30T19:29:16.745702Z\", \"duration\": \"0:05:03\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 25, \"param_--hyperparameter-criterion\": \"gini\", \"param_--hyperparameter-max_depth\": 10, \"best_metric\": 0.8064898}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1\", \"run_number\": 4, \"metric\": 0.80554944, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:30:32.593853Z\", \"end_time\": \"2021-01-30T19:35:18.975922Z\", \"created_time\": \"2021-01-30T19:29:16.836641Z\", \"created_time_dt\": \"2021-01-30T19:29:16.836641Z\", \"duration\": \"0:06:02\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 20, \"param_--hyperparameter-criterion\": \"gini\", \"param_--hyperparameter-max_depth\": 10, \"best_metric\": 0.8064898}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4\", \"run_number\": 5, \"metric\": 0.80660941, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:34:21.655367Z\", \"end_time\": \"2021-01-30T19:37:18.684449Z\", \"created_time\": \"2021-01-30T19:29:17.0152Z\", \"created_time_dt\": \"2021-01-30T19:29:17.0152Z\", \"duration\": \"0:08:01\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 20, \"param_--hyperparameter-criterion\": \"entropy\", \"param_--hyperparameter-max_depth\": 10, \"best_metric\": 0.80660941}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0\", \"run_number\": 6, \"metric\": 0.79813707, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:35:19.119477Z\", \"end_time\": \"2021-01-30T19:38:08.285852Z\", \"created_time\": \"2021-01-30T19:29:17.075305Z\", \"created_time_dt\": \"2021-01-30T19:29:17.075305Z\", \"duration\": \"0:08:51\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 15, \"param_--hyperparameter-criterion\": \"gini\", \"param_--hyperparameter-max_depth\": 10, \"best_metric\": 0.80660941}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3\", \"run_number\": 7, \"metric\": 0.81075163, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:37:27.805037Z\", \"end_time\": \"2021-01-30T19:40:22.40149Z\", \"created_time\": \"2021-01-30T19:29:17.635129Z\", \"created_time_dt\": \"2021-01-30T19:29:17.635129Z\", \"duration\": \"0:11:04\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 15, \"param_--hyperparameter-criterion\": \"entropy\", \"param_--hyperparameter-max_depth\": 10, \"best_metric\": 0.81075163}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5\", \"run_number\": 8, \"metric\": 0.80669756, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:42:38.519047Z\", \"end_time\": \"2021-01-30T19:46:01.874138Z\", \"created_time\": \"2021-01-30T19:34:54.644033Z\", \"created_time_dt\": \"2021-01-30T19:34:54.644033Z\", \"duration\": \"0:11:07\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 25, \"param_--hyperparameter-criterion\": \"entropy\", \"param_--hyperparameter-max_depth\": 10, \"best_metric\": 0.81075163}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6\", \"run_number\": 9, \"metric\": 0.8094795, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:44:37.626516Z\", \"end_time\": \"2021-01-30T19:47:40.834958Z\", \"created_time\": \"2021-01-30T19:35:56.562426Z\", \"created_time_dt\": \"2021-01-30T19:35:56.562426Z\", \"duration\": \"0:11:44\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 15, \"param_--hyperparameter-criterion\": \"gini\", \"param_--hyperparameter-max_depth\": 15, \"best_metric\": 0.81075163}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7\", \"run_number\": 10, \"metric\": 0.82436988, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:46:08.917068Z\", \"end_time\": \"2021-01-30T19:49:27.121394Z\", \"created_time\": \"2021-01-30T19:38:02.333573Z\", \"created_time_dt\": \"2021-01-30T19:38:02.333573Z\", \"duration\": \"0:11:24\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 20, \"param_--hyperparameter-criterion\": \"gini\", \"param_--hyperparameter-max_depth\": 15, \"best_metric\": 0.82436988}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8\", \"run_number\": 11, \"metric\": 0.84590861, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:47:38.077452Z\", \"end_time\": \"2021-01-30T19:51:04.464922Z\", \"created_time\": \"2021-01-30T19:39:04.339371Z\", \"created_time_dt\": \"2021-01-30T19:39:04.339371Z\", \"duration\": \"0:12:00\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 25, \"param_--hyperparameter-criterion\": \"gini\", \"param_--hyperparameter-max_depth\": 15, \"best_metric\": 0.84590861}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9\", \"run_number\": 12, \"metric\": 0.85841526, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:49:33.558599Z\", \"end_time\": \"2021-01-30T19:52:28.011242Z\", \"created_time\": \"2021-01-30T19:40:37.302834Z\", \"created_time_dt\": \"2021-01-30T19:40:37.302834Z\", \"duration\": \"0:11:50\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 15, \"param_--hyperparameter-criterion\": \"entropy\", \"param_--hyperparameter-max_depth\": 15, \"best_metric\": 0.85841526}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10\", \"run_number\": 13, \"metric\": 0.85838499, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T19:58:55.447562Z\", \"end_time\": \"2021-01-30T20:02:56.41396Z\", \"created_time\": \"2021-01-30T19:46:48.164888Z\", \"created_time_dt\": \"2021-01-30T19:46:48.164888Z\", \"duration\": \"0:16:08\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 20, \"param_--hyperparameter-criterion\": \"entropy\", \"param_--hyperparameter-max_depth\": 15, \"best_metric\": 0.85841526}, {\"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11\", \"run_number\": 14, \"metric\": 0.85420303, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2021-01-30T20:02:23.301753Z\", \"end_time\": \"2021-01-30T20:05:58.987713Z\", \"created_time\": \"2021-01-30T19:48:22.689854Z\", \"created_time_dt\": \"2021-01-30T19:48:22.689854Z\", \"duration\": \"0:17:36\", \"hyperdrive_id\": \"67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"arguments\": null, \"param_--hyperparameter-n_estimators\": 25, \"param_--hyperparameter-criterion\": \"entropy\", \"param_--hyperparameter-max_depth\": 15, \"best_metric\": 0.85841526}], \"children_metrics\": {\"categories\": [0], \"series\": {\"n_estimators\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"n_estimators\", \"stepped\": false, \"type\": \"scatter\", \"data\": [25, 20, 20, 15, 15, 25, 15, 20, 25, 15, 20, 25]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"n_estimators_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25]}], \"criterion\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"criterion\", \"stepped\": false, \"type\": \"scatter\", \"data\": [\"gini\", \"gini\", \"entropy\", \"gini\", \"entropy\", \"entropy\", \"gini\", \"gini\", \"gini\", \"entropy\", \"entropy\", \"entropy\"]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"criterion_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": []}], \"max_depth\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"max_depth\", \"stepped\": false, \"type\": \"scatter\", \"data\": [10, 10, 10, 10, 10, 10, 15, 15, 15, 15, 15, 15]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"max_depth_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [10, 10, 10, 10, 10, 10, 15, 15, 15, 15, 15, 15]}], \"min_samples_split\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"min_samples_split\", \"stepped\": false, \"type\": \"scatter\", \"data\": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"min_samples_split_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]}], \"min_samples_leaf\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"min_samples_leaf\", \"stepped\": false, \"type\": \"scatter\", \"data\": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"min_samples_leaf_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}], \"min_weight_fraction_leaf\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"min_weight_fraction_leaf\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"min_weight_fraction_leaf_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}], \"max_features\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"max_features\", \"stepped\": false, \"type\": \"scatter\", \"data\": [\"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\", \"auto\"]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"max_features_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": []}], \"min_impurity_decrease\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"min_impurity_decrease\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"min_impurity_decrease_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}], \"bootstrap\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"bootstrap\", \"stepped\": false, \"type\": \"scatter\", \"data\": [true, true, true, true, true, true, true, true, true, true, true, true]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"bootstrap_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [true, true, true, true, true, true, true, true, true, true, true, true]}], \"oob_score\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"oob_score\", \"stepped\": false, \"type\": \"scatter\", \"data\": [false, false, false, false, false, false, false, false, false, false, false, false]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"oob_score_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [false, false, false, false, false, false, false, false, false, false, false, false]}], \"n_jobs\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"n_jobs\", \"stepped\": false, \"type\": \"scatter\", \"data\": [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"n_jobs_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]}], \"random_state\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"random_state\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"random_state_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}], \"ccp_alpha\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"ccp_alpha\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"ccp_alpha_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}], \"norm_macro_recall\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"norm_macro_recall\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8064897999594313, 0.8055494387978064, 0.8066094120822759, 0.7981370660585759, 0.810751627558381, 0.8066975603367703, 0.8094794983405672, 0.8243698774225754, 0.8459086110193039, 0.858415258389476, 0.858384986884128, 0.8542030311223511]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"norm_macro_recall_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8064897999594313, 0.8064897999594313, 0.8066094120822759, 0.8066094120822759, 0.810751627558381, 0.810751627558381, 0.810751627558381, 0.8243698774225754, 0.8459086110193039, 0.858415258389476, 0.858415258389476, 0.858415258389476]}], \"model_filename\": [{\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"markers\", \"name\": \"model_filename\", \"stepped\": false, \"type\": \"scatter\", \"data\": [\"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\", \"PosixPath('outputs/model.pkl')\"]}, {\"categories\": [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], \"mode\": \"lines\", \"name\": \"model_filename_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": []}]}, \"metricName\": null, \"primaryMetricName\": \"norm_macro_recall\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"best_child_by_primary_metric\", \"run_id\": \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67\", \"categories\": [0], \"series\": [{\"data\": [{\"metric_name\": [\"norm_macro_recall\", \"norm_macro_recall\", \"norm_macro_recall\", \"norm_macro_recall\", \"norm_macro_recall\", \"norm_macro_recall\", \"norm_macro_recall\"], \"timestamp\": [\"2021-01-30 19:33:54.755569+00:00\", \"2021-01-30 19:37:06.742400+00:00\", \"2021-01-30 19:39:50.423142+00:00\", \"2021-01-30 19:48:25.821883+00:00\", \"2021-01-30 19:50:02.673394+00:00\", \"2021-01-30 19:51:38.940675+00:00\", \"2021-01-30 19:51:38.940675+00:00\"], \"run_id\": [\"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9\", \"HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9\"], \"metric_value\": [0.8064897999594313, 0.8066094120822759, 0.810751627558381, 0.8243698774225754, 0.8459086110193039, 0.858415258389476, 0.858415258389476], \"final\": [false, false, false, false, false, false, true]}]}]}], \"run_logs\": \"[2021-01-30T19:19:35.655157][API][INFO]Experiment created\\r\\n[2021-01-30T19:19:36.257793][GENERATOR][INFO]Trying to sample '5' jobs from the hyperparameter space\\r\\n[2021-01-30T19:19:36.416485][GENERATOR][INFO]Successfully sampled '5' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:19:36.8494663Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.\\r\\n[2021-01-30T19:29:15.7654599Z][SCHEDULER][INFO]The execution environment was successfully prepared.\\r\\n[2021-01-30T19:29:15.7664419Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0'\\r\\n[2021-01-30T19:29:15.9891655Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1'\\r\\n[2021-01-30T19:29:15.9108686Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3'\\r\\n[2021-01-30T19:29:15.8417821Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4'\\r\\n[2021-01-30T19:29:15.9513055Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2'\\r\\n[2021-01-30T19:29:16.8872634Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2'\\r\\n[2021-01-30T19:29:17.0642290Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1'\\r\\n[2021-01-30T19:29:17.2912767Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4'\\r\\n[2021-01-30T19:29:17.3402682Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0'\\r\\n[2021-01-30T19:29:17.8591452Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3'\\r\\n[2021-01-30T19:30:09.417204][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:30:39.719169][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:31:14.573953][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:31:44.646234][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:32:15.160268][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:32:44.833749][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:33:14.873740][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_1, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_2] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:34:26.376298][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:34:26.079413][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:34:45.661486][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:34:52.9701943Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5'\\r\\n[2021-01-30T19:34:54.8561930Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5'\\r\\n[2021-01-30T19:35:16.186888][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:35:29.846120][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:35:29.712420][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:35:47.113498][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:35:55.7894589Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6'\\r\\n[2021-01-30T19:35:56.7729288Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6'\\r\\n[2021-01-30T19:36:20.216715][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_4] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:36:51.332386][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_0] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:37:32.835100][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:37:32.966715][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:37:52.927293][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:37:59.6876478Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7'\\r\\n[2021-01-30T19:38:02.4513654Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7'\\r\\n[2021-01-30T19:38:23.132039][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:38:33.355072][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:38:33.766554][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:38:53.764670][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_3] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:39:03.7745345Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8'\\r\\n[2021-01-30T19:39:04.5332840Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8'\\r\\n[2021-01-30T19:40:35.287130][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:40:35.666120][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:40:36.3580392Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9'\\r\\n[2021-01-30T19:40:37.5293579Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9'\\r\\n[2021-01-30T19:43:01.506217][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:43:31.847791][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:44:01.786319][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:44:32.320387][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:45:03.015574][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_5, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:45:33.168681][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:46:03.768069][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:46:16.812125][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:46:16.933991][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:46:34.363889][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_6, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:46:47.0180662Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10'\\r\\n[2021-01-30T19:46:48.4175607Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10'\\r\\n[2021-01-30T19:47:05.441146][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:47:37.860199][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:47:49.522110][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2021-01-30T19:47:49.964768][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2021-01-30T19:48:08.304870][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_7, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:48:20.173395][GENERATOR][INFO]Max number of jobs '12' reached for experiment.\\r\\n[2021-01-30T19:48:20.442107][GENERATOR][INFO]All jobs generated.\\r\\n[2021-01-30T19:48:20.8866092Z][SCHEDULER][INFO]Scheduling job, id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11'\\r\\n[2021-01-30T19:48:22.8393730Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11'\\r\\n[2021-01-30T19:48:38.915700][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:49:08.930246][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:49:41.753805][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_8, https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:50:12.455356][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:50:43.374906][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:51:13.865515][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:59:22.383312][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T19:59:53.053452][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:00:26.048240][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:00:56.531750][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:01:27.424189][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:01:58.610389][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_10] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:02:28.933914][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:02:58.993745][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:03:29.361209][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:04:00.339913][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:04:35.463501][ENFORCER][INFO]Jobs [https://southcentralus.experiments.azureml.net/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experiments/**SCRUBBED**/runs/HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_11] do not contain any metrics with the primary metric name at this moment, policy cannot be applied.\\r\\n[2021-01-30T20:06:46.376285][CONTROLLER][INFO]Experiment was 'ExperimentStatus.RUNNING', is 'ExperimentStatus.FINISHED'.\\n\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.21.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "RunDetails(hyperdrive_run).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieve the best run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "gather": {
     "logged": 1598546650307
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "best_run = package_utils.get_best_run(experiment, hyperdrive_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run(Experiment: jupyter-2021-01-29-09-01,\n",
      "Id: HD_67ab13fd-0409-4b2d-807f-a943d6e47e67_9,\n",
      "Type: azureml.scriptrun,\n",
      "Status: Completed)\n",
      "{'n_estimators': 15, 'criterion': 'entropy', 'max_depth': 15, 'min_samples_split': 2, 'min_samples_leaf': 1, 'min_weight_fraction_leaf': 0.0, 'max_features': 'auto', 'min_impurity_decrease': 0.0, 'bootstrap': True, 'oob_score': False, 'n_jobs': -1, 'random_state': 0, 'ccp_alpha': 0.0, 'norm_macro_recall': 0.858415258389476, 'model_filename': \"PosixPath('outputs/model.pkl')\"}\n"
     ]
    }
   ],
   "source": [
    "print(best_run)\n",
    "print(best_run.get_metrics())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the best model locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.69 s, sys: 686 ms, total: 2.37 s\n",
      "Wall time: 21.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "best_run.download_file(\"outputs/model.pkl\", \"models/hyperdrive/model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model size in Bytes 12551631\n"
     ]
    }
   ],
   "source": [
    "print(\"model size in Bytes\", pathlib.Path(\"models/hyperdrive/model.pkl\").stat().st_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download other files from the best_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['azureml-logs/55_azureml-execution-tvmps_87f26aa19dceff6881b22262cddb911372b1b208b643e45300caadfed6f08378_p.txt', 'azureml-logs/65_job_prep-tvmps_87f26aa19dceff6881b22262cddb911372b1b208b643e45300caadfed6f08378_p.txt', 'azureml-logs/70_driver_log.txt', 'azureml-logs/75_job_post-tvmps_87f26aa19dceff6881b22262cddb911372b1b208b643e45300caadfed6f08378_p.txt', 'azureml-logs/process_info.json', 'azureml-logs/process_status.json', 'logs/azureml/79_azureml.log', 'logs/azureml/dataprep/backgroundProcess.log', 'logs/azureml/dataprep/backgroundProcess_Telemetry.log', 'logs/azureml/dataprep/engine_spans_0bdc0e2b-1e3a-4a7d-95c4-45357f0e9a6d.jsonl', 'logs/azureml/dataprep/engine_spans_7c37b2d0-3947-4335-b67f-2cbf7e5fc3b4.jsonl', 'logs/azureml/dataprep/engine_spans_fa8ec1d7-e0ad-4350-b5ac-0e6fe08fd391.jsonl', 'logs/azureml/dataprep/python_span_0bdc0e2b-1e3a-4a7d-95c4-45357f0e9a6d.jsonl', 'logs/azureml/dataprep/python_span_7c37b2d0-3947-4335-b67f-2cbf7e5fc3b4.jsonl', 'logs/azureml/dataprep/python_span_fa8ec1d7-e0ad-4350-b5ac-0e6fe08fd391.jsonl', 'logs/azureml/job_prep_azureml.log', 'logs/azureml/job_release_azureml.log', 'logs/azureml/sidecar/tvmps_87f26aa19dceff6881b22262cddb911372b1b208b643e45300caadfed6f08378_p/all.log', 'logs/azureml/sidecar/tvmps_87f26aa19dceff6881b22262cddb911372b1b208b643e45300caadfed6f08378_p/task.enter_contexts.log', 'logs/azureml/sidecar/tvmps_87f26aa19dceff6881b22262cddb911372b1b208b643e45300caadfed6f08378_p/task.exit_contexts.log', 'outputs/model.pkl']\n"
     ]
    }
   ],
   "source": [
    "print(best_run.get_file_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run.download_file(\"azureml-logs/70_driver_log.txt\",\n",
    "                       \"models/hyperdrive/70_driver_log.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explore the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 112 ms, sys: 73.7 ms, total: 186 ms\n",
      "Wall time: 238 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "fitted_model = joblib.load(\"models/hyperdrive/model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
      "                       criterion='entropy', max_depth=15, max_features='auto',\n",
      "                       max_leaf_nodes=None, max_samples=None,\n",
      "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                       min_samples_leaf=1, min_samples_split=2,\n",
      "                       min_weight_fraction_leaf=0.0, n_estimators=15, n_jobs=-1,\n",
      "                       oob_score=False, random_state=0, verbose=0,\n",
      "                       warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "print(fitted_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model on the test set from 2018 and on an additional out-of-sample test set from 2017."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model on the 2018 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 21.6 s, sys: 16.1 s, total: 37.8 s\n",
      "Wall time: 2min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test = Dataset.get_by_name(\n",
    "            workspace=workspace,\n",
    "            name=dataset_test_name,\n",
    "        ).to_pandas_dataframe()\n",
    "x_test, y_test = test.drop(labels=[\"Label\"], axis=1), test[\"Label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.2 s, sys: 1min 2s, total: 1min 12s\n",
      "Wall time: 29.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_test_predict = fitted_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          precision    recall  f1-score   support\n",
      "\n",
      "                  Benign     0.9864    0.9998    0.9930   2048549\n",
      "                     Bot     1.0000    0.9999    0.9999     43419\n",
      "        Brute Force -Web     1.0000    0.9524    0.9756        84\n",
      "        Brute Force -XSS     1.0000    1.0000    1.0000        41\n",
      "        DDOS attack-HOIC     1.0000    1.0000    1.0000     48951\n",
      "  DDoS attacks-LOIC-HTTP     1.0000    1.0000    1.0000     86440\n",
      "   DoS attacks-GoldenEye     1.0000    0.9995    0.9998      8184\n",
      "        DoS attacks-Hulk     0.9999    0.9999    0.9999      7658\n",
      "DoS attacks-SlowHTTPTest     0.7728    0.5163    0.6191     41976\n",
      "   DoS attacks-Slowloris     0.9992    1.0000    0.9996      2598\n",
      "          FTP-BruteForce     0.7177    0.8902    0.7947     58010\n",
      "           Infilteration     0.6913    0.0355    0.0675     29297\n",
      "           SQL Injection     0.8889    0.6154    0.7273        13\n",
      "          SSH-Bruteforce     1.0000    0.9998    0.9999     56208\n",
      "\n",
      "                accuracy                         0.9772   2431428\n",
      "               macro avg     0.9326    0.8578    0.8697   2431428\n",
      "            weighted avg     0.9741    0.9772    0.9714   2431428\n",
      "\n",
      "CPU times: user 3min 19s, sys: 1.62 s, total: 3min 21s\n",
      "Wall time: 3min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cr = classification_report(digits=4,\n",
    "                           y_true=y_test,\n",
    "                           y_pred=y_test_predict,\n",
    "                           output_dict=False)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model on the 2017 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.3 s, sys: 7.18 s, total: 18.4 s\n",
      "Wall time: 1min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test_2017 = Dataset.get_by_name(\n",
    "            workspace=workspace,\n",
    "            name=dataset_2017_name,\n",
    "        ).to_pandas_dataframe()\n",
    "x_test_2017, y_test_2017 = test_2017.drop(labels=[\"Label\"], axis=1), test_2017[\"Label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.14 s, sys: 16.1 s, total: 20.3 s\n",
      "Wall time: 11.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_test_2017_predict = fitted_model.predict(x_test_2017)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - CacheDriver:Cached token is expired at 2021-01-30 20:23:59.640627.  Refreshing\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - CacheDriver:Cached token is expired at 2021-01-30 20:23:59.640627.  Refreshing\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - CacheDriver:Cached token is expired at 2021-01-30 20:23:59.640627.  Refreshing\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - CacheDriver:Cached token is expired at 2021-01-30 20:23:59.640627.  Refreshing\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"aa5704da-37a8-4caa-9963-3af7d813eac6 - TokenRequest:Getting a new token from a refresh token\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          precision    recall  f1-score   support\n",
      "\n",
      "                  Benign     0.8332    0.9968    0.9076    953268\n",
      "                     Bot     0.0000    0.0000    0.0000      1956\n",
      "        Brute Force -Web     0.0000    0.0000    0.0000      1507\n",
      "        Brute Force -XSS     0.0000    0.0000    0.0000       635\n",
      "   DoS attacks-GoldenEye     0.5829    0.0460    0.0853      7714\n",
      "        DoS attacks-Hulk     0.0000    0.0000    0.0000    164013\n",
      "DoS attacks-SlowHTTPTest     0.0000    0.0000    0.0000      2327\n",
      "   DoS attacks-Slowloris     0.9405    0.7578    0.8394      4154\n",
      "          FTP-BruteForce     0.0000    0.0000    0.0000      6443\n",
      "           Infilteration     0.0000    0.0000    0.0000        33\n",
      "           SQL Injection     0.0000    0.0000    0.0000        21\n",
      "          SSH-Bruteforce     0.0004    0.0002    0.0002      5885\n",
      "\n",
      "                accuracy                         0.8308   1147956\n",
      "               macro avg     0.1964    0.1501    0.1527   1147956\n",
      "            weighted avg     0.6992    0.8308    0.7573   1147956\n",
      "\n",
      "CPU times: user 1min 28s, sys: 593 ms, total: 1min 28s\n",
      "Wall time: 1min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cr = classification_report(digits=4,\n",
    "                           y_true=y_test_2017,\n",
    "                           y_pred=y_test_2017_predict,\n",
    "                           output_dict=False)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test_2017\n",
    "del x_test_2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Model Deployment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Register the best model into the workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 267 ms, sys: 31.5 ms, total: 299 ms\n",
      "Wall time: 5.07 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = package_utils.register_model(model_name=\"hyperdrive-jupyter\",\n",
    "                                     model_path=\"outputs/model.pkl\",\n",
    "                                     run=best_run)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deploy the registered model to an Azure Container instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hyperdrive-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hyperdrive-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hyperdrive-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: hyperdrive-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f8374030250>)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f8374030250>)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f8374030250>)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f8374030250>)\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f83d00ea210>\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f83d00ea210>\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f83d00ea210>\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f83d00ea210>\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, id=hyperdrive-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, id=hyperdrive-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, id=hyperdrive-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, id=hyperdrive-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=hyperdrive-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.96 s, sys: 201 ms, total: 2.16 s\n",
      "Wall time: 15.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "service = deploy_config.main(model_name=\"hyperdrive-jupyter\",\n",
    "                             deployment_name=\"hyperdrive-jupyter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tips: You can try get_logs(): https://aka.ms/debugimage#dockerlog or local deployment: https://aka.ms/debugimage#debug-locally to debug if deployment takes longer than 10 minutes.\n",
      "Running...........................................................................................................................\n",
      "Succeeded\n",
      "ACI service creation operation finished, operation \"Succeeded\"\n",
      "CPU times: user 20 s, sys: 1.46 s, total: 21.5 s\n",
      "Wall time: 11min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "service.wait_for_deployment(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert service.state == \"Healthy\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the service endpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fetch the API keys of the service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_api_key, secondary_api_key = service.get_keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrive the scoring url of the service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://d94d9cec-8c79-4b8d-b3aa-0aa7475585d2.southcentralus.azurecontainer.io/score\n"
     ]
    }
   ],
   "source": [
    "url = service.scoring_uri\n",
    "print(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare a subset of the test dataset for submission to the service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"data\": [{\"Flow Duration\": 3761843, \"TotLen Fwd Pkts\": 1441, \"TotLen Bwd Pkts\": 1731, \"Fwd Pkt Len Std\": 191, \"Bwd Pkt Len Max\": 1179, \"Bwd Pkt Len Std\": 405, \"Flow Byts/s\": 843, \"Flow Pkts/s\": 6, \"Flow IAT Max\": 953181, \"Bwd IAT Min\": 124510, \"Bwd Header Len\": 172, \"Pkt Len Max\": 1179, \"Pkt Len Std\": 279, \"RST Flag Cnt\": 1, \"PSH Flag Cnt\": 1, \"ECE Flag Cnt\": 1, \"Init Fwd Win Byts\": 8192, \"Init Bwd Win Byts\": 62644, \"Fwd Seg Size Min\": 20}]}"
     ]
    }
   ],
   "source": [
    "input_data = json.dumps({'data': x_test[0:1].to_dict(orient='records')})\n",
    "with open(\"data.json\", \"w\") as _f:\n",
    "    _f.write(input_data)\n",
    "!cat data.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Call the service using the input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"result\": [\"Benign\"]}\n"
     ]
    }
   ],
   "source": [
    "print(service.run(input_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a post request to the service endpoint using curl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"{\\\"result\\\": [\\\"Benign\\\"]}\"CPU times: user 27.1 ms, sys: 20.9 ms, total: 48 ms\n",
      "Wall time: 1.34 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!curl -X POST \\\n",
    "      -H 'Content-Type: application/json' \\\n",
    "      -H \"Authorization: Bearer $secondary_api_key\" \\\n",
    "      --data @data.json $url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a post request to the service endpoint programatically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"result\": [\"Benign\"]}\n"
     ]
    }
   ],
   "source": [
    "# Set the content type\n",
    "headers = {\"Content-Type\": \"application/json\"}\n",
    "# If authentication is enabled, set the authorization header\n",
    "headers[\"Authorization\"] = f\"Bearer {secondary_api_key}\"\n",
    "\n",
    "resp = requests.post(url, input_data, headers=headers)\n",
    "print(resp.json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the service logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-30T20:30:46,742798867+00:00 - iot-server/run \n",
      "2021-01-30T20:30:46,743145570+00:00 - gunicorn/run \n",
      "2021-01-30T20:30:46,744434580+00:00 - rsyslog/run \n",
      "2021-01-30T20:30:46,752841946+00:00 - nginx/run \n",
      "EdgeHubConnectionString and IOTEDGE_IOTHUBHOSTNAME are not set. Exiting...\n",
      "2021-01-30T20:30:46,945078851+00:00 - iot-server/finish 1 0\n",
      "2021-01-30T20:30:46,946887265+00:00 - Exit code 1 is normal. Not restarting iot-server.\n",
      "Starting gunicorn 19.9.0\n",
      "Listening at: http://127.0.0.1:31311 (14)\n",
      "Using worker: sync\n",
      "worker timeout is set to 300\n",
      "Booting worker with pid: 43\n",
      "SPARK_HOME not set. Skipping PySpark Initialization.\n",
      "Initializing logger\n",
      "2021-01-30 20:30:47,856 | root | INFO | Starting up app insights client\n",
      "2021-01-30 20:30:47,857 | root | INFO | Starting up request id generator\n",
      "2021-01-30 20:30:47,857 | root | INFO | Starting up app insight hooks\n",
      "2021-01-30 20:30:47,857 | root | INFO | Invoking user's init function\n",
      "2021-01-30 20:30:48,207 | root | INFO | Users's init has completed successfully\n",
      "2021-01-30 20:30:48,238 | root | INFO | Skipping middleware: dbg_model_info as it's not enabled.\n",
      "2021-01-30 20:30:48,239 | root | INFO | Skipping middleware: dbg_resource_usage as it's not enabled.\n",
      "2021-01-30 20:30:48,240 | root | INFO | Scoring timeout is found from os.environ: 60000 ms\n",
      "2021-01-30 20:30:59,992 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:20:30:59 +0000] \"GET /swagger.json HTTP/1.0\" 200 3438 \"-\" \"Go-http-client/1.1\"\n",
      "2021-01-30 20:31:03,642 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:20:31:03 +0000] \"GET /swagger.json HTTP/1.0\" 200 3438 \"-\" \"Go-http-client/1.1\"\n",
      "2021-01-30 20:31:59,520 | root | INFO | Validation Request Content-Type\n",
      "2021-01-30 20:31:59,520 | root | INFO | Scoring Timer is set to 60.0 seconds\n",
      "2021-01-30 20:31:59,528 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:20:31:59 +0000] \"POST /score HTTP/1.0\" 200 28 \"-\" \"python-requests/2.24.0\"\n",
      "2021-01-30 20:32:00,762 | root | INFO | Validation Request Content-Type\n",
      "2021-01-30 20:32:00,763 | root | INFO | Scoring Timer is set to 60.0 seconds\n",
      "2021-01-30 20:32:00,774 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:20:32:00 +0000] \"POST /score HTTP/1.0\" 200 28 \"-\" \"curl/7.64.0\"\n",
      "2021-01-30 20:32:01,233 | root | INFO | Validation Request Content-Type\n",
      "2021-01-30 20:32:01,234 | root | INFO | Scoring Timer is set to 60.0 seconds\n",
      "2021-01-30 20:32:01,245 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:20:32:01 +0000] \"POST /score HTTP/1.0\" 200 28 \"-\" \"python-requests/2.24.0\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(service.get_logs())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete the service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "service.delete()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete the compute cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cluster_name starts-ws-136633\n"
     ]
    }
   ],
   "source": [
    "cluster_name = package_utils.trim_cluster_name(workspace.name)\n",
    "print(f\"cluster_name {cluster_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compute_target AmlCompute(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=starts-ws-136633, id=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633, type=AmlCompute, provisioning_state=Deleting, location=southcentralus, tags=None) is being deleted\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    compute_target = ComputeTarget(workspace=workspace, name=cluster_name)\n",
    "    compute_target.delete()\n",
    "    print(f\"compute_target {compute_target} is being deleted\")    \n",
    "except ComputeTargetException:\n",
    "    print(f\"compute_target {cluster_name} does not exist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "python3-azureml"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "nteract": {
   "version": "nteract-front-end@1.0.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}