Iqr web demo

docker/.env

@@ -0,0 +1,10 @@
+SMQTK_REGISTRY=kitware/smqtk
+
+# Dependency Image/Source versions to be used
+CUDA_DEVEL_IMAGE_TAG=9.2-cudnn7-devel-ubuntu18.04
+CUDA_RUNTIME_IMAGE_TAG=9.2-cudnn7-runtime-ubuntu18.04
+
+FAISS_VERSION=1.6.2
+
+# OUR Images
+SMQTK_TAG=latest

docker/docker-compose.build.yml

@@ -0,0 +1,105 @@
+version: "3.4"
+
+services:
+
+  #
+  # TPL: Caffe 1.0
+  #
+  tpl-caffe-base:
+    # Base stage for common use by CPU and GPU build variants.
+    image: ${SMQTK_REGISTRY}/tpl-caffe:1.0-base
+    build:
+      context: tpl/caffe1
+      target: base
+      args:
+        CUDA_DEVEL_IMAGE_TAG:  # using value from .env file.
+
+  tpl-caffe-cpu:
+    # CPU-Only build variant
+    image: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cpu
+    depends_on:
+      - tpl-caffe-base
+    build:
+      context: tpl/caffe1
+      target: caffe_cpu
+      args:
+        CUDA_DEVEL_IMAGE_TAG:  # using value from .env file.
+
+  tpl-caffe-gpu:
+    # GPU build variant
+    image: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cuda${CUDA_DEVEL_IMAGE_TAG}
+    depends_on:
+      - tpl-caffe-base
+    build:
+      context: tpl/caffe1
+      target: caffe_gpu
+      args:
+        CUDA_DEVEL_IMAGE_TAG:  # using value from .env file.
+        # One or more CUDA architecture binary versions to compile for.
+        CUDA_ARCH_BIN_LIST: 50 52 60 61 62 70 72
+
+  #
+  # TPL: FAISS
+  #
+  tpl-faiss-gpu:
+    image: ${SMQTK_REGISTRY}/tpl-faiss:${FAISS_VERSION}-cuda${CUDA_DEVEL_IMAGE_TAG}
+    build:
+      context: tpl/faiss
+      dockerfile: gpu/Dockerfile
+      args:
+        FAISS_VERSION:  # using value from .env file.
+        CUDA_DEVEL_IMAGE_TAG:  # using value from .env file.
+        # One or more CUDA architectures to compile for.
+        # Using ">-" for multi-line string concatenation with no new-lines.
+        # Using ">-" for multi-line string concatenation with no new-lines.
+        # - arch upper bound defined by CUDA version (e.g. 9.2 cannot support
+        #   compute_75)
+        # - https://docs.nvidia.com/cuda/turing-compatibility-guide/index.html#building-turing-compatible-apps-using-cuda-9-
+        # - In order to support a NEWER gpu architecture than considered here,
+        #   the final line here should specify a PTX version, e.g.
+        #   `-gencode=arch=compute_XY,code=compute_XY` (note the `code=...`),
+        #   in order to provide compatibility with future architectures.
+        CUDA_ARCH: >-
+          -gencode=arch=compute_50,code=sm_50
+          -gencode=arch=compute_52,code=sm_52
+          -gencode=arch=compute_60,code=sm_60
+          -gencode=arch=compute_61,code=sm_61
+          -gencode=arch=compute_62,code=sm_62
+          -gencode=arch=compute_70,code=sm_70
+          -gencode=arch=compute_72,code=sm_72
+          -gencode=arch=compute_72,code=compute_72
+
+  #
+  # IQR Playground Demo Image CPU/GPU
+  #
+  iqr-playground-cpu:
+    image: ${SMQTK_REGISTRY}/iqr_playground:${SMQTK_TAG}-cpu
+    depends_on:
+      - tpl-caffe-cpu
+      - tpl-faiss-gpu
+    build:
+      # Repo root
+      context: ..
+      dockerfile: docker/smqtk_iqr_playground/Dockerfile
+      target: smqtk-cpu
+      args:
+        CUDA_RUNTIME_IMAGE_TAG:  # using value from .env file.
+        CAFFE_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cpu
+        FAISS_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-faiss:${FAISS_VERSION}-cuda${CUDA_DEVEL_IMAGE_TAG}
+        FAISS_VERSION:  # using value from .env file.
+
+  iqr-playground-gpu:
+    image: ${SMQTK_REGISTRY}/iqr_playground:${SMQTK_TAG}-cuda${CUDA_RUNTIME_IMAGE_TAG}
+    depends_on:
+      - tpl-caffe-gpu
+      - tpl-faiss-gpu
+    build:
+      # Repo root
+      context: ..
+      dockerfile: docker/smqtk_iqr_playground/Dockerfile
+      target: smqtk-gpu
+      args:
+        CUDA_RUNTIME_IMAGE_TAG:  # using value from .env file.
+        CAFFE_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-caffe:1.0-cuda${CUDA_DEVEL_IMAGE_TAG}
+        FAISS_COPY_FROM_IMAGE: ${SMQTK_REGISTRY}/tpl-faiss:${FAISS_VERSION}-cuda${CUDA_DEVEL_IMAGE_TAG}
+        FAISS_VERSION:  # using value from .env file.

docker/smqtk_iqr_playground/Dockerfile

@@ -15,27 +15,29 @@ SHELL ["/bin/bash", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get -y update \
  && apt-get -y install \
-        cmake=3.10.2-1ubuntu2.18.04.1 \
+        cmake \
         curl \
-        less=487-0.1 \
-        parallel=20161222-1 \
-        python3-dev=3.6.7-1~18.04 \
+        less \
+        parallel \
+        python3-dev \
         python3-pip \
         sudo \
         vim \
+        zlib1g-dev \
+        libjpeg-dev \
         # PostgreSQL 10 Installation
-        postgresql-10=10.15-0ubuntu0.18.04.1 \
+        postgresql-10 \
         # MongoDB Installation
-        mongodb=1:3.6.3-0ubuntu1.1 \
+        mongodb \
         # Caffe/FAISS TPL deps
-        libboost-python1.65.1=1.65.1+dfsg-0ubuntu5 \
-        libboost-system1.65.1=1.65.1+dfsg-0ubuntu5 \
-        libboost-thread1.65.1=1.65.1+dfsg-0ubuntu5 \
-        libgoogle-glog0v5=0.3.5-1 \
-        libgflags2.2=2.2.1-1 \
-        libhdf5-100=1.10.0-patch1+docs-4 \
-        libprotobuf10=3.0.0-9.1ubuntu1 \
-        libopenblas-base=0.2.20+ds-4 \
+        libboost-python1.65.1 \
+        libboost-system1.65.1 \
+        libboost-thread1.65.1 \
+        libgoogle-glog0v5 \
+        libgflags2.2 \
+        libhdf5-100 \
+        libprotobuf10 \
+        libopenblas-base \
  # Clean up apt resources.
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
@@ -50,7 +52,9 @@ RUN pip3 install \
         psycopg2-binary==2.8.5 \
         # Nice-to-haves
         file-magic==0.4.0 \
-        ipython==7.14.0
+        ipython==7.14.0 \
+        jedi==0.17.2 \
+        scikit-learn==0.24.2
 
 ###############################################################################
 # COPY in FAISS wheel and install
@@ -82,27 +86,10 @@ COPY --from=caffe_image /caffe-1.0-py3-none-any.whl /
 RUN pip3 install /caffe-1.0-py3-none-any.whl \
  && rm /caffe-1.0-py3-none-any.whl
 
-###############################################################################
-# SMQTK Installation
-#
-COPY docs /smqtk/source/docs
-COPY etc /smqtk/source/etc
-COPY python /smqtk/source/python
-COPY requirements /smqtk/source/requirements
-COPY src /smqtk/source/src
-COPY TPL /smqtk/source/TPL
-COPY CMakeLists.txt LICENSE.txt pytest.* README.md setup.* setup_env.* \
-     /smqtk/source/
-RUN mkdir /smqtk/build \
- && cd /smqtk/build \
- && cmake \
-    -DCMAKE_BUILD_TYPE:STRING=Release \
-    -DCMAKE_INSTALL_PREFIX:PATH=/usr/local \
-    -DSMQTK_BUILD_FLANN:BOOL=OFF \
-    /smqtk/source \
- && make install -j$(nproc) \
- && cd / \
- && rm -rf smqtk
+# Export language options to use UTF-8, desired by Click
+ENV LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8
+
 # Export language options to use UTF-8, desired by Click
 ENV LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
@@ -112,6 +99,26 @@ ENV LC_ALL=C.UTF-8 \
 RUN ln -s /usr/share/postgresql-common/pg_wrapper /usr/local/bin/pg_ctl \
  && ln -s /usr/share/postgresql-common/pg_wrapper /usr/local/bin/postgres
 
+###############################################################################
+# SMQTK installation
+#
+RUN mkdir /smqtk/
+COPY docs /smqtk/source/docs
+COPY pyproject.toml /smqtk/source
+COPY README.md /smqtk/source
+COPY smqtk_iqr /smqtk/source/smqtk_iqr
+RUN cd /smqtk/source && \
+    pip3 install -U pip && \
+    pip3 install . && \
+    cd / && \
+    rm -rf smqtk
+
+# !!! NOTE THIS KLUDGE !!!
+# This symlink is lost during the smqtk-iqr installation, as pip currently
+# does not support symlinks, so we re-create it here.
+RUN ln -s /usr/local/lib/python3.6/dist-packages/smqtk_iqr/web/search_app/templates/base.html \
+          /usr/local/lib/python3.6/dist-packages/smqtk_iqr/web/search_app/modules/iqr/templates/base.html
+
 # Add ``smqtk`` user
 RUN useradd -mr -s /bin/bash smqtk \
  # sudo permission for modifying permissions at runtime (see entrypoint.sh).
@@ -131,19 +138,19 @@ RUN mkdir -p data/{models,configs,logs,db.psql,db.mongo} \
 
 # Configuration for Matplotlib to just use the Agg backend, not Tkinter
 COPY --chown=smqtk:smqtk \
-     devops/docker/smqtk_iqr_playground/matplotlibrc \
+     docker/smqtk_iqr_playground/matplotlibrc \
      .config/matplotlib/
 
 # Copy in general entrypoint and CPU-variant test script
 COPY --chown=smqtk:smqtk \
-     devops/docker/smqtk_iqr_playground/entrypoint.sh \
-     devops/docker/smqtk_iqr_playground/descr_comp_test.cpu.py \
+     docker/smqtk_iqr_playground/entrypoint.sh \
+     docker/smqtk_iqr_playground/descr_comp_test.cpu.py \
      /home/smqtk/
 
 # Copy in general and CPU-variant configs
 COPY --chown=smqtk:smqtk \
-     devops/docker/smqtk_iqr_playground/default_confs/* \
-     devops/docker/smqtk_iqr_playground/default_confs/cpu/* \
+     docker/smqtk_iqr_playground/default_confs/* \
+     docker/smqtk_iqr_playground/default_confs/cpu/* \
      /home/smqtk/data/configs/
 
 ENTRYPOINT ["/home/smqtk/entrypoint.sh"]
@@ -155,9 +162,9 @@ EXPOSE 5000 5001
 FROM smqtk-cpu AS smqtk-gpu
 # Add GPU-variant test script
 COPY --chown=smqtk:smqtk \
-     devops/docker/smqtk_iqr_playground/descr_comp_test.gpu.py \
+     docker/smqtk_iqr_playground/descr_comp_test.gpu.py \
      /home/smqtk/
 # Overwrite CPU-variance configs with the GPU variants.
 COPY --chown=smqtk:smqtk \
-     devops/docker/smqtk_iqr_playground/default_confs/gpu/* \
+     docker/smqtk_iqr_playground/default_confs/gpu/* \
      /home/smqtk/data/configs/

docker/smqtk_iqr_playground/default_confs/cpu/compute_many_descriptors.json

@@ -1,6 +1,6 @@
 {
     "descriptor_factory": {
-        "smqtk.representation.descriptor_element.postgres.PostgresDescriptorElement": {
+        "smqtk_descriptors.impls.descriptor_element.postgres.PostgresDescriptorElement": {
             "binary_col": "vector",
             "create_table": false,
             "db_host": "/dev/shm",
@@ -9,51 +9,50 @@
             "db_port": 5432,
             "db_user": "smqtk",
             "table_name": "descriptors_resnet50_pool5",
-            "type_col": "type_str",
             "uuid_col": "uid"
         },
-        "type": "smqtk.representation.descriptor_element.postgres.PostgresDescriptorElement"
+        "type": "smqtk_descriptors.impls.descriptor_element.postgres.PostgresDescriptorElement"
     },
     "descriptor_generator": {
-        "smqtk.algorithms.descriptor_generator.caffe_descriptor.CaffeDescriptorGenerator": {
-            "batch_size": 20,
+        "smqtk_descriptors.impls.descriptor_generator.caffe1.CaffeDescriptorGenerator": {
+            "batch_size": 10,
             "data_layer": "data",
             "gpu_device_id": 0,
             "image_mean": {
-                "smqtk.representation.data_element.file_element.DataFileElement": {
+                "smqtk_dataprovider.impls.data_element.file.DataFileElement": {
                     "explicit_mimetype": null,
                     "filepath": "/home/smqtk/caffe/msra_resnet/ResNet_mean.binaryproto",
                     "readonly": true
                 },
-                "type": "smqtk.representation.data_element.file_element.DataFileElement"
+                "type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
             },
             "input_scale": null,
             "load_truncated_images": true,
             "network_is_bgr": true,
             "network_model": {
-                "smqtk.representation.data_element.file_element.DataFileElement": {
+                "smqtk_dataprovider.impls.data_element.file.DataFileElement": {
                     "explicit_mimetype": null,
                     "filepath": "/home/smqtk/caffe/msra_resnet/ResNet-50-model.caffemodel",
                     "readonly": true
                 },
-                "type": "smqtk.representation.data_element.file_element.DataFileElement"
+                "type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
             },
             "network_prototxt": {
-                "smqtk.representation.data_element.file_element.DataFileElement": {
+                "smqtk_dataprovider.impls.data_element.file.DataFileElement": {
                     "explicit_mimetype": null,
                     "filepath": "/home/smqtk/caffe/msra_resnet/ResNet-50-deploy.prototxt",
                     "readonly": true
                 },
-                "type": "smqtk.representation.data_element.file_element.DataFileElement"
+                "type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
             },
             "pixel_rescale": null,
             "return_layer": "pool5",
             "use_gpu": false
         },
-        "type": "smqtk.algorithms.descriptor_generator.caffe_descriptor.CaffeDescriptorGenerator"
+        "type": "smqtk_descriptors.impls.descriptor_generator.caffe1.CaffeDescriptorGenerator"
     },
     "descriptor_set": {
-        "smqtk.representation.descriptor_set.postgres.PostgresDescriptorSet": {
+        "smqtk_descriptors.impls.descriptor_set.postgres.PostgresDescriptorSet": {
             "create_table": false,
             "db_host": "/dev/shm",
             "db_name": "postgres",
@@ -67,20 +66,20 @@
             "table_name": "descriptor_set_resnet50_pool5",
             "uuid_col": "uid"
         },
-        "type": "smqtk.representation.descriptor_set.postgres.PostgresDescriptorSet"
+        "type": "smqtk_descriptors.impls.descriptor_set.postgres.PostgresDescriptorSet"
     },
     "optional_data_set": {
-        "smqtk.representation.data_set.memory_set.DataMemorySet": {
+        "smqtk_dataprovider.impls.data_set.memory.DataMemorySet": {
             "cache_element": {
-                "smqtk.representation.data_element.file_element.DataFileElement": {
+                "smqtk_dataprovider.impls.data_element.file.DataFileElement": {
                     "explicit_mimetype": null,
                     "filepath": "models/image_elements.dms_cache",
                     "readonly": false
                 },
-                "type": "smqtk.representation.data_element.file_element.DataFileElement"
+                "type": "smqtk_dataprovider.impls.data_element.file.DataFileElement"
             },
             "pickle_protocol": -1
         },
-        "type": "smqtk.representation.data_set.memory_set.DataMemorySet"
+        "type": "smqtk_dataprovider.impls.data_set.memory.DataMemorySet"
     }
 }