diff --git a/03b-build-retrieval-image.ipynb b/03b-build-retrieval-image.ipynb index d444dd3..feff82c 100644 --- a/03b-build-retrieval-image.ipynb +++ b/03b-build-retrieval-image.ipynb @@ -301,7 +301,7 @@ "metadata": {}, "outputs": [], "source": [ - "# ! gcloud artifacts repositories create $REPOSITORY --repository-format=docker --location=$LOCATION" + "! gcloud artifacts repositories create $REPOSITORY --repository-format=docker --location=$LOCATION" ] }, { @@ -473,7 +473,6 @@ "outputs": [], "source": [ "# %%writefile {REPO_SRC}/.dockerignore\n", - "# *.yaml\n", "# **/__pycache__\n", "# **/*.pyc\n", "# ranking/*\n", @@ -799,12 +798,12 @@ "MODEL_ROOT_NAME = f'{APP}-{MODEL_TYPE}-{FRAMEWORK}-{VERSION}'\n", "\n", "# Docker definitions for training\n", - "IMAGE_NAME = f'{MODEL_ROOT_NAME}-tr'\n", - "IMAGE_URI = f'gcr.io/{PROJECT_ID}/{IMAGE_NAME}'\n", + "IMAGE_NAME = f'{MODEL_ROOT_NAME}-tr'\n", + "IMAGE_URI = f'gcr.io/{PROJECT_ID}/{IMAGE_NAME}'\n", "\n", - "DOCKERNAME = 'tfrs'\n", - "MACHINE_TYPE ='e2-highcpu-32'\n", - "FILE_LOCATION = \"src\" # './src'\n", + "DOCKERNAME = 'tfrs'\n", + "MACHINE_TYPE ='e2-highcpu-32'\n", + "FILE_LOCATION = \"src\" # './src'\n", "\n", "print(f\"MODEL_ROOT_NAME : {MODEL_ROOT_NAME}\")\n", "print(f\"IMAGE_URI : {IMAGE_URI}\")\n", @@ -833,10 +832,10 @@ }, "outputs": [], "source": [ - "# ! gcloud builds submit --config src/cloudbuild.yaml \\\n", - "# --substitutions _DOCKERNAME=$DOCKERNAME,_IMAGE_URI=$IMAGE_URI,_FILE_LOCATION=$FILE_LOCATION \\\n", - "# --timeout=2h \\\n", - "# --machine-type=$MACHINE_TYPE" + "! gcloud builds submit --config src/cloudbuild.yaml \\\n", + " --substitutions _DOCKERNAME=$DOCKERNAME,_IMAGE_URI=$IMAGE_URI,_FILE_LOCATION=$FILE_LOCATION \\\n", + " --timeout=2h \\\n", + " --machine-type=$MACHINE_TYPE" ] }, { diff --git a/beam_candidates/bq_to_tfr/candidate_pipeline.py b/beam_candidates/bq_to_tfr/candidate_pipeline.py index d3dac91..e586e20 100644 --- a/beam_candidates/bq_to_tfr/candidate_pipeline.py +++ b/beam_candidates/bq_to_tfr/candidate_pipeline.py @@ -1,7 +1,13 @@ +import os # import argparse import gcsfs import numpy as np +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' + +import warnings +warnings.filterwarnings('ignore') + import tensorflow as tf from datetime import datetime