clean-up-ME-indexes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2848d121-f105-4f62-8e72-5b46f739405e",
   "metadata": {},
   "source": [
    "# Clean-up Vector Search (formerly Matching Engine) Indexes and Index Endpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ae3fcb84-1f5a-492b-aced-f62e336f3b2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREFIX = ndr-v1\n"
     ]
    }
   ],
   "source": [
    "# naming convention for all cloud resources\n",
    "VERSION        = \"v1\"                  # TODO\n",
    "PREFIX         = f'ndr-{VERSION}'      # TODO\n",
    "\n",
    "print(f\"PREFIX = {PREFIX}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b7465551-247c-4484-aebb-9eebed10dfb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "PROJECT_ID               = \"hybrid-vertex\"\n",
      "PROJECT_NUM              = \"934903580331\"\n",
      "LOCATION                 = \"us-central1\"\n",
      "\n",
      "REGION                   = \"us-central1\"\n",
      "BQ_LOCATION              = \"US\"\n",
      "VPC_NETWORK_NAME         = \"ucaip-haystack-vpc-network\"\n",
      "\n",
      "VERTEX_SA                = \"934903580331-compute@developer.gserviceaccount.com\"\n",
      "\n",
      "PREFIX                   = \"ndr-v1\"\n",
      "VERSION                  = \"v1\"\n",
      "\n",
      "APP                      = \"sp\"\n",
      "MODEL_TYPE               = \"2tower\"\n",
      "FRAMEWORK                = \"tfrs\"\n",
      "DATA_VERSION             = \"v1\"\n",
      "TRACK_HISTORY            = \"5\"\n",
      "\n",
      "BUCKET_NAME              = \"ndr-v1-hybrid-vertex-bucket\"\n",
      "BUCKET_URI               = \"gs://ndr-v1-hybrid-vertex-bucket\"\n",
      "SOURCE_BUCKET            = \"spotify-million-playlist-dataset\"\n",
      "\n",
      "DATA_GCS_PREFIX          = \"data\"\n",
      "DATA_PATH                = \"gs://ndr-v1-hybrid-vertex-bucket/data\"\n",
      "VOCAB_SUBDIR             = \"vocabs\"\n",
      "VOCAB_FILENAME           = \"vocab_dict.pkl\"\n",
      "\n",
      "CANDIDATE_PREFIX         = \"candidates\"\n",
      "TRAIN_DIR_PREFIX         = \"train\"\n",
      "VALID_DIR_PREFIX         = \"valid\"\n",
      "\n",
      "VPC_NETWORK_FULL         = \"projects/934903580331/global/networks/ucaip-haystack-vpc-network\"\n",
      "\n",
      "BQ_DATASET               = \"spotify_e2e_test\"\n",
      "BQ_TABLE_TRAIN           = \"train_flatten_last_5\"\n",
      "BQ_TABLE_VALID           = \"train_flatten_valid_last_5\"\n",
      "BQ_TABLE_CANDIDATES      = \"candidates\"\n",
      "\n",
      "REPO_SRC                 = \"src\"\n",
      "PIPELINES_SUB_DIR        = \"feature_pipes\"\n",
      "\n",
      "REPOSITORY               = \"ndr-v1-spotify\"\n",
      "IMAGE_NAME               = \"train-v1\"\n",
      "REMOTE_IMAGE_NAME        = \"us-central1-docker.pkg.dev/hybrid-vertex/ndr-v1-spotify/train-v1\"\n",
      "DOCKERNAME               = \"tfrs\"\n",
      "\n",
      "SERVING_IMAGE_URI_CPU    = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-11:latest\"\n",
      "SERVING_IMAGE_URI_GPU    = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-gpu.2-11:latest\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# staging GCS\n",
    "GCP_PROJECTS             = !gcloud config get-value project\n",
    "PROJECT_ID               = GCP_PROJECTS[0]\n",
    "\n",
    "# GCS bucket and paths\n",
    "BUCKET_NAME              = f'{PREFIX}-{PROJECT_ID}-bucket'\n",
    "BUCKET_URI               = f'gs://{BUCKET_NAME}'\n",
    "\n",
    "config = !gsutil cat {BUCKET_URI}/config/notebook_env.py\n",
    "print(config.n)\n",
    "exec(config.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3d8597b7-23fb-4ac6-aab9-e1c11239a345",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import numpy as np\n",
    "\n",
    "# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' \n",
    "\n",
    "# google cloud SDKs\n",
    "from google.cloud import storage\n",
    "from google.cloud import aiplatform as vertex_ai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "14786996-636d-43f9-b7a6-9a552874a982",
   "metadata": {},
   "outputs": [],
   "source": [
    "vertex_ai.init(project=PROJECT_ID, location=LOCATION)\n",
    "\n",
    "storage_client = storage.Client(project=PROJECT_ID)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2988769-60e1-4877-b6d2-0a4773ed48b5",
   "metadata": {},
   "source": [
    "## Index Endpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9939a0f5-69d5-4846-ad02-51bb65b5fa3c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://us-central1-aiplatform.googleapis.com/]\n",
      "---\n",
      "createTime: '2023-09-25T18:17:22.236805Z'\n",
      "deployedIndexes:\n",
      "- automaticResources:\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  createTime: '2023-09-25T18:37:23.100220Z'\n",
      "  deploymentGroup: default\n",
      "  id: deployedann_v1\n",
      "  index: projects/934903580331/locations/us-central1/indexes/1713892337098162176\n",
      "  indexSyncTime: '2023-09-27T11:42:02.532637Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "description: endpoint for ann index\n",
      "displayName: ann_index_endpoint_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yN_mR5UxH4GBdKr2IVK3OQLg5Lnp-tIwgotn0cJh51YL1CjOhG6EPwlP7R5UYrC\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/7571386602446913536\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-09-25T18:17:22.946019Z'\n",
      "---\n",
      "createTime: '2023-09-25T17:53:18.383235Z'\n",
      "deployedIndexes:\n",
      "- automaticResources:\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  createTime: '2023-09-25T18:29:41.235899Z'\n",
      "  deploymentGroup: default\n",
      "  id: deployedbf_v1\n",
      "  index: projects/934903580331/locations/us-central1/indexes/1095773288241561600\n",
      "  indexSyncTime: '2023-09-27T11:43:47.333571Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "description: endpoint for brute force index\n",
      "displayName: bf_index_endpoint_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yO_zXQmzTKbCPu7ecRbey7rClnTT0MSqjpz_RmxuOye5QHp5Row-kETX1LpiwyN\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/7850046829390462976\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-09-25T17:53:18.968009Z'\n",
      "---\n",
      "createTime: '2023-09-19T16:57:46.998462Z'\n",
      "deployedIndexes:\n",
      "- automaticResources:\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  createTime: '2023-09-19T17:08:17.696381Z'\n",
      "  deploymentGroup: default\n",
      "  id: deployed_tfrs_128dim_v1_bf\n",
      "  index: projects/934903580331/locations/us-central1/indexes/3648188377053790208\n",
      "  indexSyncTime: '2023-09-27T11:43:00.741910Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "description: index endpoint for ANN index\n",
      "displayName: tfrs_128dim_v1_bf_endpoint\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yOu3cwOz6ueEQH42PCl6nBpUNUHOq6bqbJVLduCElvaQU5hSznQv42PvvmOGAtO\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/4346246319296217088\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-09-19T16:57:47.872281Z'\n",
      "---\n",
      "createTime: '2023-09-19T16:57:44.979546Z'\n",
      "deployedIndexes:\n",
      "- createTime: '2023-09-21T20:29:10.215717Z'\n",
      "  dedicatedResources:\n",
      "    machineSpec:\n",
      "      machineType: n1-standard-16\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  deploymentGroup: default\n",
      "  displayName: tfrs_128dim_v1_20230921202909\n",
      "  id: tfrs_128dim_v1_20230921202909\n",
      "  index: projects/934903580331/locations/us-central1/indexes/5708585206575792128\n",
      "  indexSyncTime: '2023-09-27T11:43:16.305348Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "- automaticResources:\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  createTime: '2023-09-19T16:58:36.365064Z'\n",
      "  deploymentGroup: default\n",
      "  id: deployed_tfrs_128dim_v1\n",
      "  index: projects/934903580331/locations/us-central1/indexes/5708585206575792128\n",
      "  indexSyncTime: '2023-09-27T11:42:59.576514Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "description: index endpoint for ANN index\n",
      "displayName: tfrs_128dim_v1_endpoint\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yM_txrTjVSa6E87MeF5eRonPmmzbxc6YA9_yJuhdrIhSaqqgn2rcuqW4YndFUU2\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/3370091100063662080\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-09-19T16:57:45.700215Z'\n",
      "---\n",
      "createTime: '2023-06-13T14:36:42.281866Z'\n",
      "deployedIndexes:\n",
      "- automaticResources:\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  createTime: '2023-06-13T14:37:34.911923Z'\n",
      "  deploymentGroup: default\n",
      "  id: deployed_ann_v14\n",
      "  index: projects/934903580331/locations/us-central1/indexes/256942673271717888\n",
      "  indexSyncTime: '2023-09-27T11:43:21.776818Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "description: endpoint for ann index\n",
      "displayName: ann_endpoint_v14\n",
      "etag: AMEw9yNNEaItfCePgPfzYyQS9uKTn0xhfoAj8XXK_XVUJLFlaW3iCX48jEM0TEGZWXgV\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/2896615004864249856\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-06-13T14:36:42.861036Z'\n",
      "---\n",
      "createTime: '2023-06-13T14:21:09.047560Z'\n",
      "deployedIndexes:\n",
      "- automaticResources:\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  createTime: '2023-06-13T14:22:00.671866Z'\n",
      "  deploymentGroup: default\n",
      "  id: deployed_bf_v14\n",
      "  index: projects/934903580331/locations/us-central1/indexes/7746428853588852736\n",
      "  indexSyncTime: '2023-09-27T11:44:28.813858Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "description: endpoint for brute force index\n",
      "displayName: bf_endpoint_v14\n",
      "etag: AMEw9yOONZ42_eNUDTdqptOVgxpLJWHu6E5RSXXsLzZYBW6vsXD6DkiMm5JG56AMqAN2\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/7369252384796573696\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-06-13T14:21:09.714484Z'\n",
      "---\n",
      "createTime: '2023-06-09T12:36:08.543508Z'\n",
      "deployedIndexes:\n",
      "- createTime: '2023-06-09T12:40:05.888834Z'\n",
      "  dedicatedResources:\n",
      "    machineSpec:\n",
      "      machineType: e2-standard-2\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  deploymentGroup: default\n",
      "  displayName: vectorstore_fly_v1_20230609124005\n",
      "  id: vectorstore_fly_v1_20230609124005\n",
      "  index: projects/934903580331/locations/us-central1/indexes/1999413516599033856\n",
      "  indexSyncTime: '2023-09-27T11:44:32.240027Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "displayName: vectorstore_fly_v1_endpoint\n",
      "etag: AMEw9yMQoH8Ws0sd1CJo3_Qp10NFNualS1F4DVZIwTTs4Ji2ek3eR9OkPTMxaELvooI=\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/8716954573287194624\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-06-09T12:36:09.211415Z'\n",
      "---\n",
      "createTime: '2023-06-06T13:58:46.511059Z'\n",
      "deployedIndexes:\n",
      "- createTime: '2023-06-06T14:01:37.094693Z'\n",
      "  dedicatedResources:\n",
      "    machineSpec:\n",
      "      machineType: e2-standard-2\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  deploymentGroup: default\n",
      "  displayName: vectorstore_thd_v1_20230606140136\n",
      "  id: vectorstore_thd_v1_20230606140136\n",
      "  index: projects/934903580331/locations/us-central1/indexes/5271278645883699200\n",
      "  indexSyncTime: '2023-09-27T11:42:53.038855Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "- createTime: '2023-06-06T14:32:29.647636Z'\n",
      "  dedicatedResources:\n",
      "    machineSpec:\n",
      "      machineType: e2-standard-2\n",
      "    maxReplicaCount: 2\n",
      "    minReplicaCount: 2\n",
      "  deploymentGroup: default\n",
      "  displayName: vectorstore_thd_v1_20230606143229\n",
      "  id: vectorstore_thd_v1_20230606143229\n",
      "  index: projects/934903580331/locations/us-central1/indexes/5271278645883699200\n",
      "  indexSyncTime: '2023-09-27T11:43:52.547835Z'\n",
      "  privateEndpoints:\n",
      "    matchGrpcAddress: 10.41.2.5\n",
      "displayName: vectorstore_thd_v1_endpoint\n",
      "etag: AMEw9yNlXJIkOcU3exdfJX0ngtVT2MrWly_J_1RIgvfN169PgLNyldRlNnxForx1gv_7\n",
      "name: projects/934903580331/locations/us-central1/indexEndpoints/3260843624727838720\n",
      "network: projects/934903580331/global/networks/ucaip-haystack-vpc-network\n",
      "updateTime: '2023-06-06T13:58:47.490810Z'\n"
     ]
    }
   ],
   "source": [
    "!gcloud beta ai index-endpoints list \\\n",
    "  --project=$PROJECT_ID \\\n",
    "  --region=$LOCATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "7acf220d-a3d8-4b15-b807-de130457458c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint object at 0x7f6e0ae3a7d0> \n",
       "resource name: projects/934903580331/locations/us-central1/indexEndpoints/1049611392061014016"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TO_DELETE = \"projects/934903580331/locations/us-central1/indexEndpoints/1049611392061014016\" # XXX\n",
    "\n",
    "my_index_endpoint = vertex_ai.MatchingEngineIndexEndpoint(index_endpoint_name=TO_DELETE)\n",
    "\n",
    "my_index_endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "328c1bb2-7f4d-4cf3-99ac-d2a0211445fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_index_endpoint.deployed_indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "98cb06c8-ccce-4cf3-9405-63625053bf4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint object at 0x7f6e0ae3a7d0> \n",
       "resource name: projects/934903580331/locations/us-central1/indexEndpoints/1049611392061014016"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_index_endpoint.undeploy_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "06b2998c-f496-47d0-807a-9e917ab0b5b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting MatchingEngineIndexEndpoint : projects/934903580331/locations/us-central1/indexEndpoints/1049611392061014016\n",
      "Delete MatchingEngineIndexEndpoint  backing LRO: projects/934903580331/locations/us-central1/operations/7749232264642297856\n",
      "MatchingEngineIndexEndpoint deleted. . Resource name: projects/934903580331/locations/us-central1/indexEndpoints/1049611392061014016\n"
     ]
    }
   ],
   "source": [
    "my_index_endpoint.delete(force=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73f50fb5-bca8-446e-8388-97a2097279cb",
   "metadata": {},
   "source": [
    "## Vector Search (Matching Engine) Indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bdbf4581-73dc-49a7-aa2f-081843feec3e",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://us-central1-aiplatform.googleapis.com/]\n",
      "---\n",
      "createTime: '2023-09-26T14:37:03.773840Z'\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_pipe-v2-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yNDevUG5nrLgk71-oji6rvQg0ym6H53FTXibd82ZhmCTc4I1fYH1Z3A3lJfhE75\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/93722371151626240\n",
      "updateTime: '2023-09-26T15:52:37.566023Z'\n",
      "---\n",
      "createTime: '2023-09-26T14:36:58.716251Z'\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: bf_index_pipe-v2_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yNHmcR1sI2YF7ZdvwJ956ehRQ_s5UjDos4VvOTL9rzn0jmwKO3rIPEc4DrFzsI=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/7936178172263923712\n",
      "updateTime: '2023-09-26T15:32:26.018514Z'\n",
      "---\n",
      "createTime: '2023-09-26T12:45:22.670487Z'\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_v1-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yNaa7Skv7ExXV8-rpAIjvaWHXiJZtTgXT8UMd-IrMyEfhbUrYTrWBOdvV4krqLG\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/8426507581693886464\n",
      "updateTime: '2023-09-26T14:00:23.826897Z'\n",
      "---\n",
      "createTime: '2023-09-26T12:43:30.806396Z'\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: bf_index_v1_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yO1njmvxXji2Bs1QMTnzHPe8JzpSMM0K8Nl5Z_keKyr2ZEf_fnKxntZ1YyIUJA=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/6164574668847054848\n",
      "updateTime: '2023-09-26T13:39:28.229093Z'\n",
      "---\n",
      "createTime: '2023-09-25T16:46:07.946644Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployedann_v1\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/7571386602446913536\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_v1-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yOWutvCDMUymbfqmQ47hlE1qYVI_p03EUFQwLJ6-FGVdeTZhLDZ5WYxtx9d4EY=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1713892337098162176\n",
      "updateTime: '2023-09-25T18:08:40.039272Z'\n",
      "---\n",
      "createTime: '2023-09-25T16:44:34.035427Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployedbf_v1\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/7850046829390462976\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: bf_index_v1_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yNKzNmTxv2SgekADHvOXCNnTMFR5af1-OxnfLctC6NQlvbhZLO2SvT38kRrzIhD\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1095773288241561600\n",
      "updateTime: '2023-09-25T17:52:05.553872Z'\n",
      "---\n",
      "createTime: '2023-09-25T16:04:16.076696Z'\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_v1-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yMNxOwE0uGKAzzMrrjFotQowcnOlNuLHQgkPyUb-9i8RPNHRVdrMWtNt_MZexXQ\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/7560690553331908608\n",
      "updateTime: '2023-09-25T17:26:32.529514Z'\n",
      "---\n",
      "createTime: '2023-09-25T16:03:12.781947Z'\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: bf_index_v1_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yP0AoDHsjNlSrlUTMeUDVeL-XCCNoVrQEJCeY2dN1U5Cw9XTY-udgMBk1a1rc2F\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/9177482819557916672\n",
      "updateTime: '2023-09-25T17:01:12.589087Z'\n",
      "---\n",
      "createTime: '2023-09-25T14:59:21.063043Z'\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_v1-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yPuM5uZHVaTbFKH8Os_tlNgVHqMUI8fLSVFzBJ_M-bP6q2aUC_xyt4yOhzuGy7y\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1797208930204516352\n",
      "updateTime: '2023-09-25T16:28:55.445267Z'\n",
      "---\n",
      "createTime: '2023-09-19T20:45:56.983425Z'\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_v1-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yOGLNhLBonSsXJDS2R3MhAeNyOp5lS8jyrGwQjxbmaBIWop64yTePF33hqwTHcu\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/5215441047378722816\n",
      "updateTime: '2023-09-19T22:09:28.766071Z'\n",
      "---\n",
      "createTime: '2023-09-19T15:23:02.158995Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: tfrs_128dim_v1_20230921202909\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3370091100063662080\n",
      "- deployedIndexId: deployed_tfrs_128dim_v1\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3370091100063662080\n",
      "description: Songs embeddings from the Spotify million playlist dataset\n",
      "displayName: tfrs_128dim_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yOgzjv9a_HSeOrB3l5tPoEq_v3j_dcyWKWm31YQG-oEbGYEjm_4Pt0IuxI06FVT\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: local-train-v1\n",
      "  experiment_run: run-20230919-150451\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/5708585206575792128\n",
      "updateTime: '2023-09-19T16:55:22.864523Z'\n",
      "---\n",
      "createTime: '2023-09-19T15:23:07.025582Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployed_tfrs_128dim_v1_bf\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/4346246319296217088\n",
      "displayName: tfrs_128dim_v1_bf\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yP8Ks1LdjxbplW0W9KfTgFu1Z9FpSTmbn_y3GR0C6Uq8vxI9D5anbtFQfiZz90D\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: local-train-v1\n",
      "  experiment_run: run-20230919-150451\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/3648188377053790208\n",
      "updateTime: '2023-09-19T16:23:35.847193Z'\n",
      "---\n",
      "createTime: '2023-06-08T16:14:23.190181Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: vectorstore_fly_v1_20230609124005\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/8716954573287194624\n",
      "description: Index for LangChain demo\n",
      "displayName: vectorstore_fly_v1\n",
      "etag: AMEw9yN__C3vgHuzkyu-dL2LMnRu1RzbWTB2RkBEWOCyShny5FH_sBYDnBIgU7JEBBI=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '5100'\n",
      "indexUpdateMethod: STREAM_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 150\n",
      "    dimensions: 768\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_SMALL\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1999413516599033856\n",
      "updateTime: '2023-06-16T14:13:06.167592Z'\n",
      "---\n",
      "createTime: '2023-06-13T13:23:38.092874Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployed_ann_v14\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/2896615004864249856\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: v1_ann_index_v14-20230613132337\n",
      "etag: AMEw9yNULzfZ0atVqjyJKvRF6kbbmBXT4gMQ8HmVmkI50LDNABv7CcTg9P1MOVFRwszt\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2183658'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/256942673271717888\n",
      "updateTime: '2023-06-13T14:35:05.866976Z'\n",
      "---\n",
      "createTime: '2023-06-13T13:23:43.945725Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployed_bf_v14\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/7369252384796573696\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: v1_bf_index_v14-20230613132343\n",
      "etag: AMEw9yNAXyC2i-WVUrRQz3vsiv4u6fKdm6-iCSoAIqxOsWyYmxjpAyIVj-Vo4d0OIR3u\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2183658'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/7746428853588852736\n",
      "updateTime: '2023-06-13T14:20:11.944970Z'\n",
      "---\n",
      "createTime: '2023-06-06T12:43:52.904135Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: vectorstore_thd_v1_20230606140136\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3260843624727838720\n",
      "- deployedIndexId: vectorstore_thd_v1_20230606143229\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3260843624727838720\n",
      "description: Index for LangChain demo\n",
      "displayName: vectorstore_thd_v1\n",
      "etag: AMEw9yMPsdJVhONpB01cIHxODmqopP21x08Pd53NqnelEQaJVm6Iojdab5Jd46eaFnH6\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '631'\n",
      "indexUpdateMethod: STREAM_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 150\n",
      "    dimensions: 768\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_SMALL\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/5271278645883699200\n",
      "updateTime: '2023-06-09T16:35:51.682791Z'\n",
      "---\n",
      "createTime: '2023-01-19T23:44:35.482073Z'\n",
      "description: Songs embeddings from the Spotify million playlist dataset\n",
      "displayName: tfrs_32dim_50e_65m\n",
      "etag: AMEw9yNGHHy1OnGHLir-Qaj1BEcimTF_vh4eYNlLQ0q1iR8bES2FqZo95qJG5tzaDMk=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2262292'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: new-50e-full-jtv12\n",
      "  experiment_run: run-20230110-150417\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 32\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/8930963516517515264\n",
      "updateTime: '2023-01-20T00:52:06.145576Z'\n",
      "---\n",
      "createTime: '2023-01-19T23:44:39.089096Z'\n",
      "displayName: tfrs_32dim_50e_65m_bf\n",
      "etag: AMEw9yPVwVhU9zRbeUrFhWK7q9QRzkNJsjgikr65BDqPtTpbOHHW6IYqB9Zrsl92GdAn\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2262292'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: new-50e-full-jtv12\n",
      "  experiment_run: run-20230110-150417\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 32\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/8006881167976431616\n",
      "updateTime: '2023-01-20T00:36:35.793930Z'\n"
     ]
    }
   ],
   "source": [
    "!gcloud ai indexes list \\\n",
    "  --project=$PROJECT_ID \\\n",
    "  --region=$LOCATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "da7a15b5-cb4e-49ab-ad80-ee3f0ae720e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<google.cloud.aiplatform.matching_engine.matching_engine_index.MatchingEngineIndex object at 0x7ff24be02510> \n",
       "resource name: projects/934903580331/locations/us-central1/indexes/5215441047378722816"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INDEX_TO_DELETE = \"projects/934903580331/locations/us-central1/indexes/5215441047378722816\" # XXXX\n",
    "\n",
    "my_index = vertex_ai.MatchingEngineIndex(index_name=INDEX_TO_DELETE)\n",
    "\n",
    "my_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f78f143e-4233-48a0-a730-b560d0c5f429",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting MatchingEngineIndex : projects/934903580331/locations/us-central1/indexes/5215441047378722816\n",
      "Delete MatchingEngineIndex  backing LRO: projects/934903580331/locations/us-central1/indexes/5215441047378722816/operations/9220730666327474176\n",
      "MatchingEngineIndex deleted. . Resource name: projects/934903580331/locations/us-central1/indexes/5215441047378722816\n"
     ]
    }
   ],
   "source": [
    "# Delete indexes\n",
    "my_index.delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "33ea78a0-2d65-49d6-8141-6db680aa79ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using endpoint [https://us-central1-aiplatform.googleapis.com/]\n",
      "---\n",
      "createTime: '2023-09-25T16:46:07.946644Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployedann_v1\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/7571386602446913536\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: ann_index_v1-v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yPJQQ9wltdwLcsWPLKHZLPsvyNStp1RIj5mD_GgxxZdR_6vxlsxxx3swUmlsxY=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1713892337098162176\n",
      "updateTime: '2023-09-25T18:08:40.039272Z'\n",
      "---\n",
      "createTime: '2023-09-25T16:44:34.035427Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployedbf_v1\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/7850046829390462976\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: bf_index_v1_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yMCTh8EA8NstuqNIBj4lEHSzhb91Gc60i_2f76H6Xtl7x62zN4xKmRJaY5t4vbG\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243058'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1095773288241561600\n",
      "updateTime: '2023-09-25T17:52:05.553872Z'\n",
      "---\n",
      "createTime: '2023-09-19T15:23:02.158995Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: tfrs_128dim_v1_20230921202909\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3370091100063662080\n",
      "- deployedIndexId: deployed_tfrs_128dim_v1\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3370091100063662080\n",
      "description: Songs embeddings from the Spotify million playlist dataset\n",
      "displayName: tfrs_128dim_v1\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yPfuP8gAuiGcSdYDlfpBAat1xkoBs0Y6KwkODrENnXjlOAhey-bpSsnnHaJrz0k\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: local-train-v1\n",
      "  experiment_run: run-20230919-150451\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/5708585206575792128\n",
      "updateTime: '2023-09-19T16:55:22.864523Z'\n",
      "---\n",
      "createTime: '2023-09-19T15:23:07.025582Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployed_tfrs_128dim_v1_bf\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/4346246319296217088\n",
      "displayName: tfrs_128dim_v1_bf\n",
      "encryptionSpec: {}\n",
      "etag: AMEw9yPMO4m3EciOKNm08l4uupJLiKtIvphpN2CjLP0JNXGMLBQoklmSUfZvWVmWMYlo\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2243497'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: local-train-v1\n",
      "  experiment_run: run-20230919-150451\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/3648188377053790208\n",
      "updateTime: '2023-09-19T16:23:35.847193Z'\n",
      "---\n",
      "createTime: '2023-06-08T16:14:23.190181Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: vectorstore_fly_v1_20230609124005\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/8716954573287194624\n",
      "description: Index for LangChain demo\n",
      "displayName: vectorstore_fly_v1\n",
      "etag: AMEw9yO3KTDF2kZgB_1KyCPatAobI00diHf8R9HJ3lzCMmSiw51ilT7qvMeizYhNgkk=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '5100'\n",
      "indexUpdateMethod: STREAM_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 150\n",
      "    dimensions: 768\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_SMALL\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/1999413516599033856\n",
      "updateTime: '2023-06-16T14:13:06.167592Z'\n",
      "---\n",
      "createTime: '2023-06-13T13:23:38.092874Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployed_ann_v14\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/2896615004864249856\n",
      "description: testing ann index for Merlin deployment\n",
      "displayName: v1_ann_index_v14-20230613132337\n",
      "etag: AMEw9yOb0yTJqmt8__nN4G2ju7qnuHDpqU8XMMOw_lNlW6U0skbOtCTmnuDVfxPLkL0I\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2183658'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/256942673271717888\n",
      "updateTime: '2023-06-13T14:35:05.866976Z'\n",
      "---\n",
      "createTime: '2023-06-13T13:23:43.945725Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: deployed_bf_v14\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/7369252384796573696\n",
      "description: testing bf index for Merlin deployment\n",
      "displayName: v1_bf_index_v14-20230613132343\n",
      "etag: AMEw9yP8DP5zoIJ52XhtfkYTxec8it3pmTcukfUiv8w8_MsOYU758bRNdGQiNq928M-t\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2183658'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 128\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/7746428853588852736\n",
      "updateTime: '2023-06-13T14:20:11.944970Z'\n",
      "---\n",
      "createTime: '2023-06-06T12:43:52.904135Z'\n",
      "deployedIndexes:\n",
      "- deployedIndexId: vectorstore_thd_v1_20230606140136\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3260843624727838720\n",
      "- deployedIndexId: vectorstore_thd_v1_20230606143229\n",
      "  indexEndpoint: projects/934903580331/locations/us-central1/indexEndpoints/3260843624727838720\n",
      "description: Index for LangChain demo\n",
      "displayName: vectorstore_thd_v1\n",
      "etag: AMEw9yMlCcrmdQMZc9kYDIeaVitdQhs4tJznQin73UYJmEDHjaTUJRjPAVcZUJ6fXDam\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '631'\n",
      "indexUpdateMethod: STREAM_UPDATE\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 150\n",
      "    dimensions: 768\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_SMALL\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/5271278645883699200\n",
      "updateTime: '2023-06-09T16:35:51.682791Z'\n",
      "---\n",
      "createTime: '2023-01-19T23:44:35.482073Z'\n",
      "description: Songs embeddings from the Spotify million playlist dataset\n",
      "displayName: tfrs_32dim_50e_65m\n",
      "etag: AMEw9yPk-aVlQyVKzzlmm63wR18rRXcevldmh-HN4ccW-ZgrPAjFf3sSKOZGHMI9Fkw=\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2262292'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: new-50e-full-jtv12\n",
      "  experiment_run: run-20230110-150417\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      treeAhConfig:\n",
      "        leafNodeEmbeddingCount: '500'\n",
      "        leafNodesToSearchPercent: 7\n",
      "    approximateNeighborsCount: 50\n",
      "    dimensions: 32\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/8930963516517515264\n",
      "updateTime: '2023-01-20T00:52:06.145576Z'\n",
      "---\n",
      "createTime: '2023-01-19T23:44:39.089096Z'\n",
      "displayName: tfrs_32dim_50e_65m_bf\n",
      "etag: AMEw9yNBk0tull_RaVz93CMYwirjeRlOzRuCJV5oEBTkLSjZ-v-IatkpPrkUCIZiSLdv\n",
      "indexStats:\n",
      "  shardsCount: 1\n",
      "  vectorsCount: '2262292'\n",
      "indexUpdateMethod: BATCH_UPDATE\n",
      "labels:\n",
      "  data_regime: full-65m\n",
      "  experiment_name: new-50e-full-jtv12\n",
      "  experiment_run: run-20230110-150417\n",
      "metadata:\n",
      "  config:\n",
      "    algorithmConfig:\n",
      "      bruteForceConfig: {}\n",
      "    dimensions: 32\n",
      "    distanceMeasureType: DOT_PRODUCT_DISTANCE\n",
      "    shardSize: SHARD_SIZE_MEDIUM\n",
      "metadataSchemaUri: gs://google-cloud-aiplatform/schema/matchingengine/metadata/nearest_neighbor_search_1.0.0.yaml\n",
      "name: projects/934903580331/locations/us-central1/indexes/8006881167976431616\n",
      "updateTime: '2023-01-20T00:36:35.793930Z'\n"
     ]
    }
   ],
   "source": [
    "!gcloud ai indexes list \\\n",
    "  --project=$PROJECT_ID \\\n",
    "  --region=$LOCATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4419dd2-696b-445f-99c3-de1d3afe5241",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7568366b-835c-49c7-8b48-453979da1bd0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-10.m103",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-10:m103"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}