03b-build-retrieval-image.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c8c695f8-f9f3-4739-b624-7dc2d8973351",
   "metadata": {},
   "source": [
    "## Build custom container for Vertex training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "aa9157c3-4189-4ca2-b210-874540cc85f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jupyter/jw-repo2/spotify_mpd_two_tower\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a13c7445-9e33-4852-89be-6a5eb397dda4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREFIX = ndr-v1\n"
     ]
    }
   ],
   "source": [
    "# naming convention for all cloud resources\n",
    "VERSION        = \"v1\"                  # TODO\n",
    "PREFIX         = f'ndr-{VERSION}'      # TODO\n",
    "\n",
    "print(f\"PREFIX = {PREFIX}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1dcebfde-8981-43aa-805b-0e8520fef5e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "PROJECT_ID               = \"hybrid-vertex\"\n",
      "PROJECT_NUM              = \"934903580331\"\n",
      "LOCATION                 = \"us-central1\"\n",
      "\n",
      "REGION                   = \"us-central1\"\n",
      "BQ_LOCATION              = \"US\"\n",
      "VPC_NETWORK_NAME         = \"ucaip-haystack-vpc-network\"\n",
      "\n",
      "VERTEX_SA                = \"934903580331-compute@developer.gserviceaccount.com\"\n",
      "\n",
      "PREFIX                   = \"ndr-v1\"\n",
      "VERSION                  = \"v1\"\n",
      "\n",
      "APP                      = \"sp\"\n",
      "MODEL_TYPE               = \"2tower\"\n",
      "FRAMEWORK                = \"tfrs\"\n",
      "DATA_VERSION             = \"v1\"\n",
      "TRACK_HISTORY            = \"5\"\n",
      "\n",
      "BUCKET_NAME              = \"ndr-v1-hybrid-vertex-bucket\"\n",
      "BUCKET_URI               = \"gs://ndr-v1-hybrid-vertex-bucket\"\n",
      "SOURCE_BUCKET            = \"spotify-million-playlist-dataset\"\n",
      "\n",
      "DATA_GCS_PREFIX          = \"data\"\n",
      "DATA_PATH                = \"gs://ndr-v1-hybrid-vertex-bucket/data\"\n",
      "VOCAB_SUBDIR             = \"vocabs\"\n",
      "VOCAB_FILENAME           = \"vocab_dict.pkl\"\n",
      "\n",
      "CANDIDATE_PREFIX         = \"candidates\"\n",
      "TRAIN_DIR_PREFIX         = \"train\"\n",
      "VALID_DIR_PREFIX         = \"valid\"\n",
      "\n",
      "VPC_NETWORK_FULL         = \"projects/934903580331/global/networks/ucaip-haystack-vpc-network\"\n",
      "\n",
      "BQ_DATASET               = \"spotify_e2e_test\"\n",
      "BQ_TABLE_TRAIN           = \"train_flatten_last_5\"\n",
      "BQ_TABLE_VALID           = \"train_flatten_valid_last_5\"\n",
      "BQ_TABLE_CANDIDATES      = \"candidates\"\n",
      "\n",
      "REPO_SRC                 = \"src\"\n",
      "PIPELINES_SUB_DIR        = \"feature_pipes\"\n",
      "\n",
      "REPOSITORY               = \"ndr-v1-spotify\"\n",
      "IMAGE_NAME               = \"train-v1\"\n",
      "REMOTE_IMAGE_NAME        = \"us-central1-docker.pkg.dev/hybrid-vertex/ndr-v1-spotify/train-v1\"\n",
      "DOCKERNAME               = \"tfrs\"\n",
      "\n",
      "SERVING_IMAGE_URI_CPU    = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-11:latest\"\n",
      "SERVING_IMAGE_URI_GPU    = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-11:latest\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# staging GCS\n",
    "GCP_PROJECTS             = !gcloud config get-value project\n",
    "PROJECT_ID               = GCP_PROJECTS[0]\n",
    "\n",
    "# GCS bucket and paths\n",
    "BUCKET_NAME              = f'{PREFIX}-{PROJECT_ID}-bucket'\n",
    "BUCKET_URI               = f'gs://{BUCKET_NAME}'\n",
    "\n",
    "config = !gsutil cat {BUCKET_URI}/config/notebook_env.py\n",
    "print(config.n)\n",
    "exec(config.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b318b5d4-417a-483f-b9d1-2a1e071e16de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://ndr-v1-hybrid-vertex-bucket/config/\n",
      "gs://ndr-v1-hybrid-vertex-bucket/data/\n",
      "gs://ndr-v1-hybrid-vertex-bucket/endpoint-tests/\n",
      "gs://ndr-v1-hybrid-vertex-bucket/local-train-v1/\n",
      "gs://ndr-v1-hybrid-vertex-bucket/local-train-v2/\n",
      "gs://ndr-v1-hybrid-vertex-bucket/scale-training-v1/\n",
      "gs://ndr-v1-hybrid-vertex-bucket/tfrs-pipe-v1/\n"
     ]
    }
   ],
   "source": [
    "! gsutil ls $BUCKET_URI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46c76aa3-cd22-410f-86a9-24978985e0aa",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3f858913-b13c-4d5d-ba54-24c36b0365c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7390306-48b3-4020-aab9-0de0c5b72c75",
   "metadata": {},
   "source": [
    "# Build Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d6f423c0-5bd5-401a-aa77-f14205ca9603",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DOCKERNAME        = tfrs\n",
      "REPOSITORY        = ndr-v1-spotify\n",
      "IMAGE_NAME        = train-v1\n",
      "REMOTE_IMAGE_NAME = us-central1-docker.pkg.dev/hybrid-vertex/ndr-v1-spotify/train-v1\n"
     ]
    }
   ],
   "source": [
    "print(f\"DOCKERNAME        = {DOCKERNAME}\")\n",
    "print(f\"REPOSITORY        = {REPOSITORY}\")\n",
    "print(f\"IMAGE_NAME        = {IMAGE_NAME}\")\n",
    "print(f\"REMOTE_IMAGE_NAME = {REMOTE_IMAGE_NAME}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "99098cc3-982e-4e53-9ca5-5cc868cd376f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jupyter/jw-repo2/spotify_mpd_two_tower\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "50dc7713-07f5-4fd5-8448-10b5fca82cd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[01;34msrc\u001b[00m\n",
      "├── Dockerfile_rank\n",
      "├── Dockerfile_tfrs\n",
      "├── cloudbuild.yaml\n",
      "├── \u001b[01;34mfeature_pipes\u001b[00m\n",
      "│   ├── call_spotify_api_artist.py\n",
      "│   └── call_spotify_api_audio.py\n",
      "├── \u001b[01;34mfeatures\u001b[00m\n",
      "│   └── feature_store_configs.py\n",
      "├── \u001b[01;34mranking\u001b[00m\n",
      "│   ├── __init__.py\n",
      "│   ├── build_audio_ranker.py\n",
      "│   ├── feature_sets.py\n",
      "│   ├── requirements.txt\n",
      "│   ├── task.py\n",
      "│   ├── tf_ranking_model.py\n",
      "│   ├── train_config.py\n",
      "│   └── train_utils.py\n",
      "├── \u001b[01;34mtrain_pipes\u001b[00m\n",
      "│   ├── create_ann_index.py\n",
      "│   ├── create_ann_index_endpoint_vpc.py\n",
      "│   ├── create_brute_force_index.py\n",
      "│   ├── create_brute_index_endpoint_vpc.py\n",
      "│   ├── create_tensorboard.py\n",
      "│   ├── deploy_ann_index.py\n",
      "│   ├── deploy_brute_index.py\n",
      "│   ├── generate_candidates.py\n",
      "│   ├── model_monitoring_config.py\n",
      "│   ├── pipeline_config.py\n",
      "│   ├── test_model_endpoint.py\n",
      "│   ├── test_model_index_endpoint.py\n",
      "│   └── train_custom_model.py\n",
      "├── \u001b[01;34mtwo_tower_jt\u001b[00m\n",
      "│   ├── __init__.py\n",
      "│   ├── feature_sets.py\n",
      "│   ├── interactive_train.py\n",
      "│   ├── requirements.txt\n",
      "│   ├── task.py\n",
      "│   ├── test_instances.py\n",
      "│   ├── train_config.py\n",
      "│   ├── train_utils.py\n",
      "│   └── two_tower.py\n",
      "└── \u001b[01;34mvocab_pipes\u001b[00m\n",
      "    ├── adapt_fixed_text_layer_vocab.py\n",
      "    ├── adapt_ragged_text_layer_vocab.py\n",
      "    ├── config.py\n",
      "    └── create_master_vocab.py\n",
      "\n",
      "6 directories, 40 files\n"
     ]
    }
   ],
   "source": [
    "!tree src"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "818ecf36-0652-4f7a-b26c-6992ae931c68",
   "metadata": {},
   "source": [
    "## Create Artifact Repository\n",
    "If you don't have an existing artifact repository, create one using the gcloud command below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9ae3bd0d-efdc-4060-9ef5-0cf5ee947580",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! gcloud artifacts repositories create $REPOSITORY --repository-format=docker --location=$LOCATION"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04a29ccf-9cf5-4234-bf1d-08a8b9fcc7ab",
   "metadata": {},
   "source": [
    "## Local Docker build\n",
    "Provide a name for your dockerfile and make sure you are authenticated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "04d30d5d-8fd0-4ad3-a1c4-e54f64498a62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33mWARNING:\u001b[0m Your config file at [/home/jupyter/.docker/config.json] contains these credential helper entries:\n",
      "\n",
      "{\n",
      "  \"credHelpers\": {\n",
      "    \"gcr.io\": \"gcloud\",\n",
      "    \"us.gcr.io\": \"gcloud\",\n",
      "    \"eu.gcr.io\": \"gcloud\",\n",
      "    \"asia.gcr.io\": \"gcloud\",\n",
      "    \"staging-k8s.gcr.io\": \"gcloud\",\n",
      "    \"marketplace.gcr.io\": \"gcloud\",\n",
      "    \"us-central1-docker.pkg.dev\": \"gcloud\"\n",
      "  }\n",
      "}\n",
      "Adding credentials for: us-central1-docker.pkg.dev\n",
      "gcloud credential helpers already registered correctly.\n"
     ]
    }
   ],
   "source": [
    "! gcloud auth configure-docker $REGION-docker.pkg.dev --quiet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdbfe299-4060-4c58-8102-d8c0bcdc2b4f",
   "metadata": {},
   "source": [
    "### Requirements & Config files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "14a89b21-1ec4-44d4-8707-9d445169fe1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/two_tower_jt/train_config.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile {REPO_SRC}/two_tower_jt/train_config.py\n",
    "PROJECT_ID='hybrid-vertex'\n",
    "TRACK_HISTORY = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0ae3cae3-54e3-4e22-ace8-8c12d1b697e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/two_tower_jt/requirements.txt\n"
     ]
    }
   ],
   "source": [
    "%%writefile {REPO_SRC}/two_tower_jt/requirements.txt\n",
    "google-cloud-aiplatform[cloud_profiler]>=1.26.1\n",
    "tensorflow-recommenders==0.7.2\n",
    "tensorboard==2.10.1\n",
    "# tensorboard==2.11.2 \n",
    "tensorboard-data-server==0.6.1\n",
    "tensorboard-plugin-profile==2.11.1\n",
    "tensorflow-io==0.27.0\n",
    "tensorboard-plugin-wit\n",
    "protobuf==3.19.6"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c91324a5-14b4-4126-ae35-e8c88e287111",
   "metadata": {},
   "source": [
    "## Create Dockerfile"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ccdbf4f-b41b-49bf-9185-935111d22abe",
   "metadata": {},
   "source": [
    "* do this : `COPY /dir /dir`\n",
    "* not this: `COPY /dir/* /dir/`\n",
    "\n",
    "commands\n",
    "* `ENV PYTHONDONTWRITEBYTECODE 1`\n",
    "* alt_base_image: `FROM gcr.io/deeplearning-platform-release/tf-gpu.2-11`\n",
    "* `COPY /src/two_tower_jt $APP_HOME/src/two_tower_jt`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0811a484-33f2-47a0-b22e-6764d2156d7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/Dockerfile_tfrs\n"
     ]
    }
   ],
   "source": [
    "%%writefile {REPO_SRC}/Dockerfile_{DOCKERNAME}\n",
    "\n",
    "FROM tensorflow/tensorflow:2.11.0-gpu\n",
    "\n",
    "ENV PYTHONUNBUFFERED True\n",
    "\n",
    "ENV APP_HOME /workspace\n",
    "\n",
    "WORKDIR $APP_HOME\n",
    "\n",
    "COPY src/two_tower_jt $APP_HOME/src/two_tower_jt\n",
    "\n",
    "RUN pip install --upgrade pip\n",
    "RUN pip install -r $APP_HOME/src/two_tower_jt/requirements.txt\n",
    "\n",
    "RUN apt update && apt -y install nvtop\n",
    "\n",
    "RUN export PYTHONPATH=${PYTHONPATH}:${APP_HOME}/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eefcbb35-da05-4559-a47a-d88ea9fcf3ce",
   "metadata": {},
   "source": [
    "## .dockerignore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c6617e81-58f2-4f9a-934f-f4b70f3c350a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%writefile {REPO_SRC}/.dockerignore\n",
    "# **/__pycache__\n",
    "# **/*.pyc\n",
    "# ranking/*\n",
    "# feature_pipes/*\n",
    "# vocab_pipes/*"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8b49d20-6e34-442a-a3ca-9868beffadb5",
   "metadata": {},
   "source": [
    "## Build Image Locally"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b85992f2-02ff-4d4b-b1cc-16c9bf75d74c",
   "metadata": {},
   "source": [
    "run commands in terminal window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "177d09ef-bd88-4154-a687-651ec28d89e8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copy these commands into terminal:\n",
      "\n",
      "export REPO_SRC=src\n",
      "export REMOTE_IMAGE_NAME=us-central1-docker.pkg.dev/hybrid-vertex/ndr-v1-spotify/train-v1\n",
      "export DOCKERNAME=tfrs\n",
      "docker build -t $REMOTE_IMAGE_NAME -f $REPO_SRC/Dockerfile_$DOCKERNAME .\n"
     ]
    }
   ],
   "source": [
    "# # set variables if running in terminal\n",
    "print(\"copy these commands into terminal:\\n\")\n",
    "print(f\"export REPO_SRC={REPO_SRC}\")\n",
    "print(f\"export REMOTE_IMAGE_NAME={REMOTE_IMAGE_NAME}\")\n",
    "print(f\"export DOCKERNAME={DOCKERNAME}\")\n",
    "print(f\"docker build -t $REMOTE_IMAGE_NAME -f $REPO_SRC/Dockerfile_$DOCKERNAME .\")\n",
    "\n",
    "# !docker build -t $REMOTE_IMAGE_NAME -f $REPO_SRC/Dockerfile_$DOCKERNAME ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ca2aa78-db96-441c-81e6-1fd074410e2b",
   "metadata": {},
   "source": [
    "## Push container to Registry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "670737e4-a802-4799-b462-bff489e23c79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copy this command into terminal:\n",
      "\n",
      "docker push $REMOTE_IMAGE_NAME\n"
     ]
    }
   ],
   "source": [
    "print(\"copy this command into terminal:\\n\")\n",
    "print(f\"docker push $REMOTE_IMAGE_NAME\")\n",
    "\n",
    "# ! docker push $REMOTE_IMAGE_NAME"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae61c63f-ccd3-4878-87d4-2f01f8632479",
   "metadata": {},
   "source": [
    "```\n",
    "sudo apt-get autoclean\n",
    "sudo apt-get autoremove\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6654bb4-786e-4051-877c-b656bf7482b8",
   "metadata": {
    "tags": []
   },
   "source": [
    "## (Optional) Build Image with Cloud Build"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "da3e40ef-2059-45b7-a86c-148c077833f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/jupyter/jw-repo2/spotify_mpd_two_tower'"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# root_path = '..'\n",
    "root_path = '/home/jupyter/jw-repo2/spotify_mpd_two_tower'\n",
    "os.chdir(root_path)\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "68f308bb-129e-4fb2-ab94-156950889618",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! gcloud beta artifacts repositories create {REPOSITORY} \\\n",
    "#     --repository-format=docker \\\n",
    "#     --location=$LOCATION"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e8d35db-6052-48dc-b374-2120fd726558",
   "metadata": {},
   "source": [
    "### set `gcloudignore`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "26ce47d0-9ddd-426c-97fd-1f6e42a7f5ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated property [gcloudignore/enabled].\n"
     ]
    }
   ],
   "source": [
    "! gcloud config set gcloudignore/enabled true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "f990929a-df64-4844-94bd-d61da1ac8c0f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting .gcloudignore\n"
     ]
    }
   ],
   "source": [
    "%%writefile .gcloudignore\n",
    ".gcloudignore\n",
    "img/*\n",
    "*.pkl\n",
    "*.png\n",
    "*.ipynb\n",
    ".git\n",
    ".github\n",
    ".ipynb_checkpoints/*\n",
    "candidate_embeddings.json\n",
    "spotipy_secret_creds.py\n",
    "pip_freeze.txt\n",
    "custom_pipeline_spec.json\n",
    "prediction_featurestore_fetch_config.yaml\n",
    "README.md\n",
    ".gitignore\n",
    ".DS_Store\n",
    "WIP/two_tower_lite.py\n",
    "WIP/ranking/ranking.md\n",
    "beam_candidates/*\n",
    "beam_training/*\n",
    "learning/*\n",
    "train_pipes/*\n",
    "vocab_pipes/*\n",
    "ARCHIVED/*\n",
    "ranking/*\n",
    "util/__pycache__/*\n",
    "src/.dockerignore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "b56d89e8-c1eb-4d53-a264-6df5b238da22",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "src/Dockerfile_tfrs\n",
      "src/cloudbuild.yaml\n",
      "src/Dockerfile_rank\n",
      "src/features/feature_store_configs.py\n",
      "src/two_tower_jt/__init__.py\n",
      "src/two_tower_jt/train_utils.py\n",
      "src/two_tower_jt/interactive_train.py\n",
      "src/two_tower_jt/test_instances.py\n",
      "src/two_tower_jt/train_config.py\n",
      "src/two_tower_jt/two_tower.py\n",
      "src/two_tower_jt/requirements.txt\n",
      "src/two_tower_jt/feature_sets.py\n",
      "src/two_tower_jt/task.py\n",
      "src/feature_pipes/call_spotify_api_audio.py\n",
      "src/feature_pipes/call_spotify_api_artist.py\n",
      "util/local_utils.py\n",
      "util/feature_set_utils.py\n",
      "util/two_tower_16_bit_edition.py\n",
      "util/train_utils.py\n",
      "util/install-nvtop.sh\n",
      "util/accelerators.py\n",
      "util/workerpool_specs.py\n",
      "util/test_instances.py\n"
     ]
    }
   ],
   "source": [
    "!gcloud meta list-files-for-upload"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a084e68-458e-40a2-a841-7f030d73c8c5",
   "metadata": {},
   "source": [
    "### submit job to Cloud Build"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "8b253bee-5f24-472b-abc8-795241afce91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting src/cloudbuild.yaml\n"
     ]
    }
   ],
   "source": [
    "%%writefile {REPO_SRC}/cloudbuild.yaml\n",
    "\n",
    "steps:\n",
    "- name: 'gcr.io/cloud-builders/docker'\n",
    "  args: ['build', '-t', '$_IMAGE_URI', '$_FILE_LOCATION', '-f', '$_FILE_LOCATION/Dockerfile_$_DOCKERNAME']\n",
    "images:\n",
    "- '$_IMAGE_URI'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "13fa9d0e-e611-49b0-ad30-0aaff1eb39dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "8f12bb30-807e-4661-9c72-96420519983f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL_ROOT_NAME   : sp-2tower-tfrs-v1\n",
      "IMAGE_URI         : gcr.io/hybrid-vertex/sp-2tower-tfrs-v1-tr\n",
      "DOCKERNAME        : tfrs\n",
      "IMAGE_URI         : gcr.io/hybrid-vertex/sp-2tower-tfrs-v1-tr\n",
      "FILE_LOCATION     : src\n",
      "MACHINE_TYPE      : e2-highcpu-32\n"
     ]
    }
   ],
   "source": [
    "MODEL_ROOT_NAME         = f'{APP}-{MODEL_TYPE}-{FRAMEWORK}-{VERSION}'\n",
    "\n",
    "# Docker definitions for training\n",
    "IMAGE_NAME              = f'{MODEL_ROOT_NAME}-tr'\n",
    "IMAGE_URI               = f'gcr.io/{PROJECT_ID}/{IMAGE_NAME}'\n",
    "\n",
    "DOCKERNAME              = 'tfrs'\n",
    "MACHINE_TYPE            ='e2-highcpu-32'\n",
    "FILE_LOCATION           = \"src\" # './src'\n",
    "\n",
    "print(f\"MODEL_ROOT_NAME   : {MODEL_ROOT_NAME}\")\n",
    "print(f\"IMAGE_URI         : {IMAGE_URI}\")\n",
    "print(f\"DOCKERNAME        : {DOCKERNAME}\")\n",
    "print(f\"IMAGE_URI         : {IMAGE_URI}\")\n",
    "print(f\"FILE_LOCATION     : {FILE_LOCATION}\")\n",
    "print(f\"MACHINE_TYPE      : {MACHINE_TYPE}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "5501d625-7aa9-45b1-844b-0cafda6a45f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !tree src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "041ce55b-3cea-41f7-9046-3ab8100e2beb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "! gcloud builds submit --config src/cloudbuild.yaml \\\n",
    "    --substitutions _DOCKERNAME=$DOCKERNAME,_IMAGE_URI=$IMAGE_URI,_FILE_LOCATION=$FILE_LOCATION \\\n",
    "    --timeout=2h \\\n",
    "    --machine-type=$MACHINE_TYPE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f58dfbc1-b639-4c97-9823-427fcee5fa8c",
   "metadata": {},
   "source": [
    "**Finished**"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-10.m103",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-10:m103"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}