Skip to content

Commit

Permalink
Merge pull request #10 from joshanderson-kw/iqr_demo
Browse files Browse the repository at this point in the history
Iqr web demo
  • Loading branch information
Purg authored Nov 12, 2021
2 parents 358323d + 53d3e86 commit 88375e5
Show file tree
Hide file tree
Showing 32 changed files with 1,419 additions and 299 deletions.
10 changes: 10 additions & 0 deletions docker/.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
SMQTK_REGISTRY=kitware/smqtk

# Dependency Image/Source versions to be used
CUDA_DEVEL_IMAGE_TAG=9.2-cudnn7-devel-ubuntu18.04
CUDA_RUNTIME_IMAGE_TAG=9.2-cudnn7-runtime-ubuntu18.04

FAISS_VERSION=1.6.2

# OUR Images
SMQTK_TAG=latest
105 changes: 105 additions & 0 deletions docker/docker-compose.build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
version: "3.4"

services:

#
# TPL: Caffe 1.0
#
tpl-caffe-base:
# Base stage for common use by CPU and GPU build variants.
image: ${SMQTK_REGISTRY}/tpl-caffe:1.0-base
build:
context: tpl/caffe1
target: base
args:
CUDA_DEVEL_IMAGE_TAG: # using value from .env file.

tpl-caffe-cpu:
# CPU-Only build variant
image: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cpu
depends_on:
- tpl-caffe-base
build:
context: tpl/caffe1
target: caffe_cpu
args:
CUDA_DEVEL_IMAGE_TAG: # using value from .env file.

tpl-caffe-gpu:
# GPU build variant
image: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cuda${CUDA_DEVEL_IMAGE_TAG}
depends_on:
- tpl-caffe-base
build:
context: tpl/caffe1
target: caffe_gpu
args:
CUDA_DEVEL_IMAGE_TAG: # using value from .env file.
# One or more CUDA architecture binary versions to compile for.
CUDA_ARCH_BIN_LIST: 50 52 60 61 62 70 72

#
# TPL: FAISS
#
tpl-faiss-gpu:
image: ${SMQTK_REGISTRY}/tpl-faiss:${FAISS_VERSION}-cuda${CUDA_DEVEL_IMAGE_TAG}
build:
context: tpl/faiss
dockerfile: gpu/Dockerfile
args:
FAISS_VERSION: # using value from .env file.
CUDA_DEVEL_IMAGE_TAG: # using value from .env file.
# One or more CUDA architectures to compile for.
# Using ">-" for multi-line string concatenation with no new-lines.
# Using ">-" for multi-line string concatenation with no new-lines.
# - arch upper bound defined by CUDA version (e.g. 9.2 cannot support
# compute_75)
# - https://docs.nvidia.com/cuda/turing-compatibility-guide/index.html#building-turing-compatible-apps-using-cuda-9-
# - In order to support a NEWER gpu architecture than considered here,
# the final line here should specify a PTX version, e.g.
# `-gencode=arch=compute_XY,code=compute_XY` (note the `code=...`),
# in order to provide compatibility with future architectures.
CUDA_ARCH: >-
-gencode=arch=compute_50,code=sm_50
-gencode=arch=compute_52,code=sm_52
-gencode=arch=compute_60,code=sm_60
-gencode=arch=compute_61,code=sm_61
-gencode=arch=compute_62,code=sm_62
-gencode=arch=compute_70,code=sm_70
-gencode=arch=compute_72,code=sm_72
-gencode=arch=compute_72,code=compute_72
#
# IQR Playground Demo Image CPU/GPU
#
iqr-playground-cpu:
image: ${SMQTK_REGISTRY}/iqr_playground:${SMQTK_TAG}-cpu
depends_on:
- tpl-caffe-cpu
- tpl-faiss-gpu
build:
# Repo root
context: ..
dockerfile: docker/smqtk_iqr_playground/Dockerfile
target: smqtk-cpu
args:
CUDA_RUNTIME_IMAGE_TAG: # using value from .env file.
CAFFE_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cpu
FAISS_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-faiss:${FAISS_VERSION}-cuda${CUDA_DEVEL_IMAGE_TAG}
FAISS_VERSION: # using value from .env file.

iqr-playground-gpu:
image: ${SMQTK_REGISTRY}/iqr_playground:${SMQTK_TAG}-cuda${CUDA_RUNTIME_IMAGE_TAG}
depends_on:
- tpl-caffe-gpu
- tpl-faiss-gpu
build:
# Repo root
context: ..
dockerfile: docker/smqtk_iqr_playground/Dockerfile
target: smqtk-gpu
args:
CUDA_RUNTIME_IMAGE_TAG: # using value from .env file.
CAFFE_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cuda${CUDA_DEVEL_IMAGE_TAG}
FAISS_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-faiss:${FAISS_VERSION}-cuda${CUDA_DEVEL_IMAGE_TAG}
FAISS_VERSION: # using value from .env file.
93 changes: 50 additions & 43 deletions docker/smqtk_iqr_playground/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,29 @@ SHELL ["/bin/bash", "-c"]
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get -y update \
&& apt-get -y install \
cmake=3.10.2-1ubuntu2.18.04.1 \
cmake \
curl \
less=487-0.1 \
parallel=20161222-1 \
python3-dev=3.6.7-1~18.04 \
less \
parallel \
python3-dev \
python3-pip \
sudo \
vim \
zlib1g-dev \
libjpeg-dev \
# PostgreSQL 10 Installation
postgresql-10=10.15-0ubuntu0.18.04.1 \
postgresql-10 \
# MongoDB Installation
mongodb=1:3.6.3-0ubuntu1.1 \
mongodb \
# Caffe/FAISS TPL deps
libboost-python1.65.1=1.65.1+dfsg-0ubuntu5 \
libboost-system1.65.1=1.65.1+dfsg-0ubuntu5 \
libboost-thread1.65.1=1.65.1+dfsg-0ubuntu5 \
libgoogle-glog0v5=0.3.5-1 \
libgflags2.2=2.2.1-1 \
libhdf5-100=1.10.0-patch1+docs-4 \
libprotobuf10=3.0.0-9.1ubuntu1 \
libopenblas-base=0.2.20+ds-4 \
libboost-python1.65.1 \
libboost-system1.65.1 \
libboost-thread1.65.1 \
libgoogle-glog0v5 \
libgflags2.2 \
libhdf5-100 \
libprotobuf10 \
libopenblas-base \
# Clean up apt resources.
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand All @@ -50,7 +52,9 @@ RUN pip3 install \
psycopg2-binary==2.8.5 \
# Nice-to-haves
file-magic==0.4.0 \
ipython==7.14.0
ipython==7.14.0 \
jedi==0.17.2 \
scikit-learn==0.24.2

###############################################################################
# COPY in FAISS wheel and install
Expand Down Expand Up @@ -82,27 +86,10 @@ COPY --from=caffe_image /caffe-1.0-py3-none-any.whl /
RUN pip3 install /caffe-1.0-py3-none-any.whl \
&& rm /caffe-1.0-py3-none-any.whl

###############################################################################
# SMQTK Installation
#
COPY docs /smqtk/source/docs
COPY etc /smqtk/source/etc
COPY python /smqtk/source/python
COPY requirements /smqtk/source/requirements
COPY src /smqtk/source/src
COPY TPL /smqtk/source/TPL
COPY CMakeLists.txt LICENSE.txt pytest.* README.md setup.* setup_env.* \
/smqtk/source/
RUN mkdir /smqtk/build \
&& cd /smqtk/build \
&& cmake \
-DCMAKE_BUILD_TYPE:STRING=Release \
-DCMAKE_INSTALL_PREFIX:PATH=/usr/local \
-DSMQTK_BUILD_FLANN:BOOL=OFF \
/smqtk/source \
&& make install -j$(nproc) \
&& cd / \
&& rm -rf smqtk
# Export language options to use UTF-8, desired by Click
ENV LC_ALL=C.UTF-8 \
LANG=C.UTF-8

# Export language options to use UTF-8, desired by Click
ENV LC_ALL=C.UTF-8 \
LANG=C.UTF-8
Expand All @@ -112,6 +99,26 @@ ENV LC_ALL=C.UTF-8 \
RUN ln -s /usr/share/postgresql-common/pg_wrapper /usr/local/bin/pg_ctl \
&& ln -s /usr/share/postgresql-common/pg_wrapper /usr/local/bin/postgres

###############################################################################
# SMQTK installation
#
RUN mkdir /smqtk/
COPY docs /smqtk/source/docs
COPY pyproject.toml /smqtk/source
COPY README.md /smqtk/source
COPY smqtk_iqr /smqtk/source/smqtk_iqr
RUN cd /smqtk/source && \
pip3 install -U pip && \
pip3 install . && \
cd / && \
rm -rf smqtk

# !!! NOTE THIS KLUDGE !!!
# This symlink is lost during the smqtk-iqr installation, as pip currently
# does not support symlinks, so we re-create it here.
RUN ln -s /usr/local/lib/python3.6/dist-packages/smqtk_iqr/web/search_app/templates/base.html \
/usr/local/lib/python3.6/dist-packages/smqtk_iqr/web/search_app/modules/iqr/templates/base.html

# Add ``smqtk`` user
RUN useradd -mr -s /bin/bash smqtk \
# sudo permission for modifying permissions at runtime (see entrypoint.sh).
Expand All @@ -131,19 +138,19 @@ RUN mkdir -p data/{models,configs,logs,db.psql,db.mongo} \

# Configuration for Matplotlib to just use the Agg backend, not Tkinter
COPY --chown=smqtk:smqtk \
devops/docker/smqtk_iqr_playground/matplotlibrc \
docker/smqtk_iqr_playground/matplotlibrc \
.config/matplotlib/

# Copy in general entrypoint and CPU-variant test script
COPY --chown=smqtk:smqtk \
devops/docker/smqtk_iqr_playground/entrypoint.sh \
devops/docker/smqtk_iqr_playground/descr_comp_test.cpu.py \
docker/smqtk_iqr_playground/entrypoint.sh \
docker/smqtk_iqr_playground/descr_comp_test.cpu.py \
/home/smqtk/

# Copy in general and CPU-variant configs
COPY --chown=smqtk:smqtk \
devops/docker/smqtk_iqr_playground/default_confs/* \
devops/docker/smqtk_iqr_playground/default_confs/cpu/* \
docker/smqtk_iqr_playground/default_confs/* \
docker/smqtk_iqr_playground/default_confs/cpu/* \
/home/smqtk/data/configs/

ENTRYPOINT ["/home/smqtk/entrypoint.sh"]
Expand All @@ -155,9 +162,9 @@ EXPOSE 5000 5001
FROM smqtk-cpu AS smqtk-gpu
# Add GPU-variant test script
COPY --chown=smqtk:smqtk \
devops/docker/smqtk_iqr_playground/descr_comp_test.gpu.py \
docker/smqtk_iqr_playground/descr_comp_test.gpu.py \
/home/smqtk/
# Overwrite CPU-variance configs with the GPU variants.
COPY --chown=smqtk:smqtk \
devops/docker/smqtk_iqr_playground/default_confs/gpu/* \
docker/smqtk_iqr_playground/default_confs/gpu/* \
/home/smqtk/data/configs/
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"descriptor_factory": {
"smqtk.representation.descriptor_element.postgres.PostgresDescriptorElement": {
"smqtk_descriptors.impls.descriptor_element.postgres.PostgresDescriptorElement": {
"binary_col": "vector",
"create_table": false,
"db_host": "/dev/shm",
Expand All @@ -9,51 +9,50 @@
"db_port": 5432,
"db_user": "smqtk",
"table_name": "descriptors_resnet50_pool5",
"type_col": "type_str",
"uuid_col": "uid"
},
"type": "smqtk.representation.descriptor_element.postgres.PostgresDescriptorElement"
"type": "smqtk_descriptors.impls.descriptor_element.postgres.PostgresDescriptorElement"
},
"descriptor_generator": {
"smqtk.algorithms.descriptor_generator.caffe_descriptor.CaffeDescriptorGenerator": {
"batch_size": 20,
"smqtk_descriptors.impls.descriptor_generator.caffe1.CaffeDescriptorGenerator": {
"batch_size": 10,
"data_layer": "data",
"gpu_device_id": 0,
"image_mean": {
"smqtk.representation.data_element.file_element.DataFileElement": {
"smqtk_dataprovider.impls.data_element.file.DataFileElement": {
"explicit_mimetype": null,
"filepath": "/home/smqtk/caffe/msra_resnet/ResNet_mean.binaryproto",
"readonly": true
},
"type": "smqtk.representation.data_element.file_element.DataFileElement"
"type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
},
"input_scale": null,
"load_truncated_images": true,
"network_is_bgr": true,
"network_model": {
"smqtk.representation.data_element.file_element.DataFileElement": {
"smqtk_dataprovider.impls.data_element.file.DataFileElement": {
"explicit_mimetype": null,
"filepath": "/home/smqtk/caffe/msra_resnet/ResNet-50-model.caffemodel",
"readonly": true
},
"type": "smqtk.representation.data_element.file_element.DataFileElement"
"type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
},
"network_prototxt": {
"smqtk.representation.data_element.file_element.DataFileElement": {
"smqtk_dataprovider.impls.data_element.file.DataFileElement": {
"explicit_mimetype": null,
"filepath": "/home/smqtk/caffe/msra_resnet/ResNet-50-deploy.prototxt",
"readonly": true
},
"type": "smqtk.representation.data_element.file_element.DataFileElement"
"type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
},
"pixel_rescale": null,
"return_layer": "pool5",
"use_gpu": false
},
"type": "smqtk.algorithms.descriptor_generator.caffe_descriptor.CaffeDescriptorGenerator"
"type": "smqtk_descriptors.impls.descriptor_generator.caffe1.CaffeDescriptorGenerator"
},
"descriptor_set": {
"smqtk.representation.descriptor_set.postgres.PostgresDescriptorSet": {
"smqtk_descriptors.impls.descriptor_set.postgres.PostgresDescriptorSet": {
"create_table": false,
"db_host": "/dev/shm",
"db_name": "postgres",
Expand All @@ -67,20 +66,20 @@
"table_name": "descriptor_set_resnet50_pool5",
"uuid_col": "uid"
},
"type": "smqtk.representation.descriptor_set.postgres.PostgresDescriptorSet"
"type": "smqtk_descriptors.impls.descriptor_set.postgres.PostgresDescriptorSet"
},
"optional_data_set": {
"smqtk.representation.data_set.memory_set.DataMemorySet": {
"smqtk_dataprovider.impls.data_set.memory.DataMemorySet": {
"cache_element": {
"smqtk.representation.data_element.file_element.DataFileElement": {
"smqtk_dataprovider.impls.data_element.file.DataFileElement": {
"explicit_mimetype": null,
"filepath": "models/image_elements.dms_cache",
"readonly": false
},
"type": "smqtk.representation.data_element.file_element.DataFileElement"
"type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
},
"pickle_protocol": -1
},
"type": "smqtk.representation.data_set.memory_set.DataMemorySet"
"type": "smqtk_dataprovider.impls.data_set.memory.DataMemorySet"
}
}
Loading

0 comments on commit 88375e5

Please sign in to comment.