diff --git a/00-env-setup.ipynb b/00-env-setup.ipynb index 64a2aa6..349cb77 100644 --- a/00-env-setup.ipynb +++ b/00-env-setup.ipynb @@ -103,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "c9bd35d9-6c0b-40b9-873c-5b137c62db33", "metadata": {}, "outputs": [], @@ -117,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "b2833324-082c-470d-916a-db133c2ce53d", "metadata": {}, "outputs": [ @@ -137,6 +137,9 @@ "VALID_DIR_PREFIX : valid\n", "VPC_NETWORK_FULL : projects/934903580331/global/networks/ucaip-haystack-vpc-network\n", "BQ_DATASET : spotify_e2e_test\n", + "BQ_TABLE_TRAIN : v2_train_flatten_last_5\n", + "BQ_TABLE_VALID : v2_train_flatten_valid_last_5\n", + "BQ_TABLE_CANDIDATES : candidates\n", "REPO_SRC : src\n", "PIPELINES_SUB_DIR : feature_pipes\n", "REPOSITORY : ndr-v1-spotify\n", @@ -168,6 +171,9 @@ "\n", "# BigQuery parameters\n", "BQ_DATASET = 'spotify_e2e_test'\n", + "BQ_TABLE_TRAIN = 'v2_train_flatten_last_5'\n", + "BQ_TABLE_VALID = 'v2_train_flatten_valid_last_5'\n", + "BQ_TABLE_CANDIDATES = 'candidates'\n", "\n", "# repo\n", "REPO_SRC = 'src'\n", @@ -199,6 +205,9 @@ "print(f\"VPC_NETWORK_FULL : {VPC_NETWORK_FULL}\")\n", "\n", "print(f\"BQ_DATASET : {BQ_DATASET}\")\n", + "print(f\"BQ_TABLE_TRAIN : {BQ_TABLE_TRAIN}\")\n", + "print(f\"BQ_TABLE_VALID : {BQ_TABLE_VALID}\")\n", + "print(f\"BQ_TABLE_CANDIDATES : {BQ_TABLE_CANDIDATES}\")\n", "\n", "print(f\"REPO_SRC : {REPO_SRC}\")\n", "print(f\"PIPELINES_SUB_DIR : {PIPELINES_SUB_DIR}\")\n", @@ -214,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "3c9aeab8-73aa-4503-b91b-3ab7902596b1", "metadata": {}, "outputs": [ @@ -234,7 +243,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "3c69c1a3-ee9a-4b94-b67a-f38b05b60eef", "metadata": {}, "outputs": [], @@ -254,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "01b46ec0-998e-42f2-8aa2-ec907ec2838c", "metadata": {}, "outputs": [ @@ -298,6 +307,9 @@ "VPC_NETWORK_FULL = \"projects/934903580331/global/networks/ucaip-haystack-vpc-network\"\n", "\n", "BQ_DATASET = \"spotify_e2e_test\"\n", + "BQ_TABLE_TRAIN = \"v2_train_flatten_last_5\"\n", + "BQ_TABLE_VALID = \"v2_train_flatten_valid_last_5\"\n", + "BQ_TABLE_CANDIDATES = \"candidates\"\n", "\n", "REPO_SRC = \"src\"\n", "PIPELINES_SUB_DIR = \"feature_pipes\"\n", @@ -350,6 +362,9 @@ "VPC_NETWORK_FULL = \\\"{VPC_NETWORK_FULL}\\\"\n", "\n", "BQ_DATASET = \\\"{BQ_DATASET}\\\"\n", + "BQ_TABLE_TRAIN = \\\"{BQ_TABLE_TRAIN}\\\"\n", + "BQ_TABLE_VALID = \\\"{BQ_TABLE_VALID}\\\"\n", + "BQ_TABLE_CANDIDATES = \\\"{BQ_TABLE_CANDIDATES}\\\"\n", "\n", "REPO_SRC = \\\"{REPO_SRC}\\\"\n", "PIPELINES_SUB_DIR = \\\"{PIPELINES_SUB_DIR}\\\"\n", @@ -367,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "87901338-bccd-41f0-8aeb-edad2b0a91f7", "metadata": {}, "outputs": [ @@ -387,7 +402,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "cad49778-eb36-4b20-9201-3a266a97dbc8", "metadata": {}, "outputs": [ @@ -396,7 +411,10 @@ "output_type": "stream", "text": [ "gs://ndr-v1-hybrid-vertex-bucket/config/\n", - "gs://ndr-v1-hybrid-vertex-bucket/data/\n" + "gs://ndr-v1-hybrid-vertex-bucket/data/\n", + "gs://ndr-v1-hybrid-vertex-bucket/local-train-v1/\n", + "gs://ndr-v1-hybrid-vertex-bucket/scale-training-v1/\n", + "gs://ndr-v1-hybrid-vertex-bucket/tfrs-pipe-v1/\n" ] } ], diff --git a/02-tfrecord-beam-pipeline.ipynb b/02-tfrecord-beam-pipeline.ipynb index 0a0d3c3..e1daaad 100644 --- a/02-tfrecord-beam-pipeline.ipynb +++ b/02-tfrecord-beam-pipeline.ipynb @@ -125,9 +125,9 @@ "metadata": {}, "outputs": [], "source": [ - "BQ_TABLE_TRAIN = 'v2_train_flatten_last_5'\n", - "BQ_TABLE_VALID = 'v2_train_flatten_valid_last_5'\n", - "BQ_TABLE_CANDIDATES = 'candidates'\n", + "# BQ_TABLE_TRAIN = 'v2_train_flatten_last_5'\n", + "# BQ_TABLE_VALID = 'v2_train_flatten_valid_last_5'\n", + "# BQ_TABLE_CANDIDATES = 'candidates'\n", "\n", "# =============================== #\n", "# included in env-setup.ipynb #\n", diff --git a/src/train_pipes/test_model_index_endpoint.py b/src/train_pipes/test_model_index_endpoint.py index cf15a47..d966a3b 100644 --- a/src/train_pipes/test_model_index_endpoint.py +++ b/src/train_pipes/test_model_index_endpoint.py @@ -15,7 +15,7 @@ 'numpy' ], ) -def test_model_index_endpoint_v5( +def test_model_index_endpoint( project: str, location: str, version: str, diff --git a/src/vetex/MatchingEngineCRUD.py b/src/vertex/MatchingEngineCRUD.py similarity index 100% rename from src/vetex/MatchingEngineCRUD.py rename to src/vertex/MatchingEngineCRUD.py diff --git a/src/vetex/__init__.py b/src/vertex/__init__.py similarity index 100% rename from src/vetex/__init__.py rename to src/vertex/__init__.py diff --git a/src/vetex/helpers.py b/src/vertex/helpers.py similarity index 100% rename from src/vetex/helpers.py rename to src/vertex/helpers.py