From 2793f7a99143e1cdf6051ccb797474b5fcd180fd Mon Sep 17 00:00:00 2001
From: Steve Schmerler <git@elcorto.com>
Date: Sun, 3 Sep 2017 17:46:24 +0200
Subject: [PATCH] API: use Keras and NNs, remove cmd line interface

Complete rewrite. We now use keras and pre-trained NNs for calculating
fingerprints. The command line interface is not necessary, we use this
package as a library only.
---
 .gitignore                           |   3 +
 README.rst                           | 268 +++++++++++++--------------
 bin/00resize.py                      | 111 -----------
 bin/10fingerprints.py                |  54 ------
 bin/20cluster.py                     |  64 -------
 imagecluster/__init__.py             |   0
 {imgcmp => imagecluster}/env.py      |   0
 imagecluster/imagecluster.py         | 146 +++++++++++++++
 imagecluster/main.py                 |  26 +++
 imagecluster/test/test.py            |  39 ++++
 imgcmp.egg-info/PKG-INFO             | 173 -----------------
 imgcmp.egg-info/SOURCES.txt          |  14 --
 imgcmp.egg-info/dependency_links.txt |   1 -
 imgcmp.egg-info/top_level.txt        |   1 -
 imgcmp/__init__.py                   |   1 -
 imgcmp/calc.py                       |  83 ---------
 imgcmp/cli.py                        |  11 --
 imgcmp/io.py                         |  26 ---
 imgcmp/misc.py                       |  25 ---
 imgcmp/tests/test_idx.py             |  22 ---
 setup.py                             |  17 +-
 21 files changed, 350 insertions(+), 735 deletions(-)
 delete mode 100755 bin/00resize.py
 delete mode 100755 bin/10fingerprints.py
 delete mode 100755 bin/20cluster.py
 create mode 100644 imagecluster/__init__.py
 rename {imgcmp => imagecluster}/env.py (100%)
 create mode 100644 imagecluster/imagecluster.py
 create mode 100644 imagecluster/main.py
 create mode 100644 imagecluster/test/test.py
 delete mode 100644 imgcmp.egg-info/PKG-INFO
 delete mode 100644 imgcmp.egg-info/SOURCES.txt
 delete mode 100644 imgcmp.egg-info/dependency_links.txt
 delete mode 100644 imgcmp.egg-info/top_level.txt
 delete mode 100644 imgcmp/__init__.py
 delete mode 100644 imgcmp/calc.py
 delete mode 100644 imgcmp/cli.py
 delete mode 100644 imgcmp/io.py
 delete mode 100644 imgcmp/misc.py
 delete mode 100644 imgcmp/tests/test_idx.py

diff --git a/.gitignore b/.gitignore
index 0d20b64..9c17578 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,4 @@
 *.pyc
+*.egg-info
+*.swp
+*.bak
diff --git a/README.rst b/README.rst
index e044ae9..194bb1f 100644
--- a/README.rst
+++ b/README.rst
@@ -1,162 +1,148 @@
 About
 =====
-Package for comparing images by content. Uses simple image hashing
-("fingerprints"). See "Methods" below for details.
+
+Package for comparing and clustering images by content. We use a pre-trained
+deep convolutional neural network for calculating image fingerprints, which are
+then used to cluster similar images.
 
 Install
 =======
 
 ::
 
-    git clone ...
-    pip3 install -e .
-
-Usage
-=====
-
-There are 3 scripts in ``bin/``::
-
-    00resize.py  10fingerprints.py  20cluster.py
-
-These must be executed one after another. Have a look at the help (use
-``<script> -h``).
-
-The first one, resizing images, is optional. However, if you have large images
-(several MB/image), you should really resize them first in order to lower the
-computational cost of subsequent operations. This makes sense if you want to
-play with various parameters of the clustering.
-
-::
-
-    $ ./bin/00resize.py 0.2 /path/to/large/pics/* -vv
+    $ git clone https://github.com/elcorto/imagecluster.git
+    $ pip3 install -e .
 
-This will resize all images to 20% (factor 0.2) of their original size. Resized
-files are written to ``~/.imgcmp/convert/`` by default. Now, calculate the
-fingerprint database::
+or::
 
-    $ ./bin/10fingerprints.py 
+    $ python3 setup build --prefix=...
 
-This uses all files in ``~/.imgcmp/convert/*`` by default. Alternatively,
-specify all images on the command line. It creates a file
-``~/.imgcmp/fingerprints.hdf`` (HDF5 file format, fingerprint database). 
-
-Last, we will cluster the images (actually their fingerprints) by using a
-similarity index ( ``0..1``, ``0.3`` below). The index can be thought of as the
-allowed *dissimilarity*. A small index means to put only very similar images in
-one cluster. The extreme case 0.0 means to allow zero dissimilarity and thus
-put each image in a cluster of size 1. In contrast, large values imply less
-strict clustering and will put more but less similar images in a cluster. A
-value of 1.0 is equal to putting all images in one single cluster (e.g. all
-images are treated as equal)::
-
-    $ ./bin/20cluster.py 0.3
-    cluster dir: /home/user/.imgcmp/cluster
-    items per cluster : number of such clusters
-    2 : 41
-    3 : 2
-    4 : 2
-
-By default, the database ``~/.imgcmp/fingerprints.hdf`` is used. Have a look at
-the clusters (as dirs with symlinks to the relevant files)::
-
-    $ ls ~/.imgcmp/cluster/
-    cluster_with_2  cluster_with_3  cluster_with_4
-
-    $ ls ~/.imgcmp/cluster/cluster_with_2/
-    cluster_0   cluster_1  cluster_2
+Usage
+=====
 
-    $ qiv -ifm ~/.imgcmp/cluster/cluster_with_2/cluster_0/
+See ``imagecluster.main.main()`` for a usage example.
+
+If there is no fingerprints database, it will first run all images through the
+NN model and calculate fingerprints. Then it will cluster the images based on
+the fingerprints and a similarity index (more details below).
+
+Example session::
+
+	>>> from imagecluster import main
+	>>> main.main('/path/to/testpics/', sim=0.5)
+	no fingerprints database /path/to/testpics/fingerprints.pk found
+	running all images thru NN model ...
+	/path/to/testpics/DSC_1061.JPG
+	/path/to/testpics/DSC_1080.JPG
+	...
+	/path/to/testpics/DSC_1087.JPG
+	clustering ...
+	cluster dir: /path/to/testpics/clusters
+	items per cluster : number of such clusters
+	2 : 7
+	3 : 2
+	4 : 4
+	5 : 1
+	10 : 1
+
+Have a look at the clusters (as dirs with symlinks to the relevant files)::
+
+	$ tree /path/to/testpics
+	/path/to/testpics/clusters
+	├── cluster_with_10
+	│   └── cluster_0
+	│       ├── DSC_1068.JPG -> /path/to/testpics/DSC_1068.JPG
+	│       ├── DSC_1070.JPG -> /path/to/testpics/DSC_1070.JPG
+	│       ├── DSC_1071.JPG -> /path/to/testpics/DSC_1071.JPG
+	│       ├── DSC_1072.JPG -> /path/to/testpics/DSC_1072.JPG
+	│       ├── DSC_1073.JPG -> /path/to/testpics/DSC_1073.JPG
+	│       ├── DSC_1074.JPG -> /path/to/testpics/DSC_1074.JPG
+	│       ├── DSC_1075.JPG -> /path/to/testpics/DSC_1075.JPG
+	│       ├── DSC_1076.JPG -> /path/to/testpics/DSC_1076.JPG
+	│       ├── DSC_1077.JPG -> /path/to/testpics/DSC_1077.JPG
+	│       └── DSC_1078.JPG -> /path/to/testpics/DSC_1078.JPG
+	├── cluster_with_2
+	│   ├── cluster_0
+	│   │   ├── DSC_1037.JPG -> /path/to/testpics/DSC_1037.JPG
+	│   │   └── DSC_1038.JPG -> /path/to/testpics/DSC_1038.JPG
+	│   ├── cluster_1
+	│   │   ├── DSC_1053.JPG -> /path/to/testpics/DSC_1053.JPG
+	│   │   └── DSC_1054.JPG -> /path/to/testpics/DSC_1054.JPG
+	│   ├── cluster_2
+	│   │   ├── DSC_1046.JPG -> /path/to/testpics/DSC_1046.JPG
+	│   │   └── DSC_1047.JPG -> /path/to/testpics/DSC_1047.JPG
+	...
 
 Methods
 =======
 
-What we can do and what not
----------------------------
-
-We use a variant of the "phash" method -- a simple and fast way to calculate
-fingerprints. The package can detect images which are rather similar, e.g. the
-same scene photographed twice or more with some camera movement in between, or
-a scene with the same background and e.g. one person exchanged. Good parameter
-values are x=8 ... 16 (``10fingerprints.py -x``) and similarity index sim=0.1
-... 0.3 (``20cluster.py <sim>``). However, values above sim=0.3 will quickly
-recognize all sorts of images as equal and usually, results obtained with sim >
-0.5 are seldom useful. Next, the higher x, the more detailed and close to the
-original image the fingerprint is, since the original is squashed down to a x
-times x grayscale image, converted to binary (pixed is black or white), and
-flattened into a vector of x**2 length (the "fingerprint"). High values such as
-64 already result in detection of only extremely similar images, since too much
-information of the original image is kept. Instead, a value of x=2 is obviously
-useless because it reduces each image to a 2x2 matrix (and fingerprint vector of
-length 4) with almost no information from the original image left. So x=8 and
-sim=0.1 is roughly equivalent to x=16 and sim=0.3. Therefore, if you get too
-many "false positives" (images counted as similar when they are not), reduce
-sim or increase x.
+Image fingerprints
+------------------
 
 The original goal was to have a clustering based on classification of image
 *content* -- smth like: image A this an image of my kitchen; image B is also an
 image of my kitchen, only from a different angle and some persons in the
 foreground, but the information -- this is my kitchen -- is the same. This is a
-feature-detection task which relies on the ability to recognize *objects*
-within a scene, regardless of other scene parameters (like view angle, color,
-light, ...). It turns out that we may need Neural Networks and some real
-machine learning for the generation of better *feature vectors*, e.g. a feature
-vector that always encodes the information "my kitchen" -- similar to DNNs
-which learn features automatically. The simple image hashing done here is
-rather limited in that respect. It only does a very pedestrian smoothing /
-low-pass filtering to reduce the noise and extract the "important" parts of the
-image. But this helps to find duplicates and almost-duplicates in a collection
-of photos. 
-
-Note that we do not want to do the clustering with an NN, only the feature
-vector generation. The clustering shall be done in a classical fashion as used
-here (hierarchical/agglomerative clustering).
-
-NN-related keywords for doing it right:
-
-* distributed representation/embedding: the network has an abstract
-  representation of the object ("my kitchen"), distributed across the whole net
-  via the weights
-* unsupervised learning: learn that all images should produce the same net
-  output "my kitchen" w/o labeled training data -- i.e. find clusters
-  (structure in the data) automatically
-* neighorhood component analysis, wasabi, category learning
-
-image fingerprints: simple and fast
------------------------------------
-These methods basically squash down the image to something like 16x16,
-transform to gray scale and store that as a feature vector of length 16x16, for
-example -> fast. But the method is not invariant against rotation and the
-like.
-
-The idea is always to calculate a database of image fingerprints ("hashes",
-feature vectors) and then do searches in feature space (all fingerprints) using
-some form of KD-tree / nearest neighbor search.
-
-* google: calculate image fingerprint
-* [a|p|d]hash:
-  https://realpython.com/blog/python/fingerprinting-images-for-near-duplicate-detection/ 
-* especially: phash.org
-* older Perl implementation of a ahash(?)-like method:
-  http://www.jhnc.org/findimagedupes/manpage.html, also as Debian package
-
-more scientific "feature extraction"
-------------------------------------
-
-* classical CV (computer vision): SIFT (good but slow, old-school
-  hand-engineered feature detector), SURF (faster version of
-  SIFT)
-* http://opencv-python-tutroals.readthedocs.org/en/latest/index.html
-    * SIFT and SURF are patented, so use ORB instead
-      http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_feature2d/py_orb/py_orb.html#orb
-* opencv Bag Of Words: http://stackoverflow.com/questions/7205489/opencv-fingerprint-image-and-compare-against-database
-
-Python image processing
------------------------
-* google: python image processing :)
-* http://scikit-image.org/
-* PIL vs. Pillow: http://docs.python-guide.org/en/latest/scenarios/imaging/
-* http://www.scipy-lectures.org/advanced/image_processing
-
-better methods
---------------
-read about: Content-based image classification
+feature-detection task which relies on the ability to recognize then content of
+the scene, regardless of other scene parameters (like view angle, color, light,
+...). It turns out that we can use deep convolutional neural networks
+(convnets) for the generation of good *feature vectors*, e.g. a feature vector
+that always encodes the information "my kitchen", since deep nets, once trained
+on many different images, have developed an internal representation of objects
+like chair, boat, car .. and kitchen. Simple image hashing, which we used
+previously, is rather limited in that respect. It only does a very pedestrian
+smoothing / low-pass filtering to reduce the noise and extract the "important"
+parts of the image. This helps to find duplicates and almost-duplicates in a
+collection of photos. 
+
+To this end, we use a pre-trained NN (VGG16_ as implemented by Keras_). The
+network was trained on ImageNet_ and is able to categorize images in to 1000
+classes (the last layer has 1000 nodes). We chop off the last layer (`thanks
+for the hint! <alexcnwy_>`_) and use the activations of the pre-last fully
+connected layer (4096 nodes) as image fingerprints (numpy 1d array of shape
+``(4096,)``).
+
+The package can detect images which are rather similar, e.g. the same scene
+photographed twice or more with some camera movement in between, or a scene
+with the same background and e.g. one person exchanged. This was also possible
+with image hashes. 
+
+Now with NN-based fingerprints, we also cluster all sorts of images which have,
+e.g. mountains, tents, or beaches, so this is far better. However, if you run
+this on a large collection of images which contain images with tents or
+beaches, then the system won't recognize that certain images belong together
+because they were taken on the same trip. All tent images will be in one
+cluster, and so will all beaches images. This is probably b/c in this case, the
+classification of the image happens by looking at the background. A tent in the
+center of the image will always look the same, but it is the background which
+makes humans distinguish the context. The problem is: VGG16 and all the other
+popular networks have been trained on ridiculously small images of 224x224 size
+because of computational limitations, where it is impossible to recognize
+background details.
+
+Clustering
+----------
+
+We use hierarchical clustering. This is basic stuff straight from scipy. See
+``imagecluster.calc.cluster()``. The image fingerprints (4096-dim vectors) are
+compared using a distance metric and similar images are put together in a
+cluster. The threshold for what counts as similar is defined by a similar index
+(again, see ``calc.cluster()``).
+
+The index can be thought of as the allowed *dissimilarity* or a similarity
+tolerance. A small index means to put only very similar images in one cluster.
+The extreme case 0.0 means to allow zero dissimilarity and thus to put each image
+in a cluster of size 1. In contrast, large values imply less strict clustering
+and will put more but less similar images in a cluster. A value of 1.0 is equal
+to putting all images in one single cluster (all images are treated as
+equal).
+
+Tests
+=====
+
+Run ``nosetests3`` (nosetests for Python3, Linux).
+
+.. _VGG16: https://arxiv.org/abs/1409.1556
+.. _Keras: https://keras.io
+.. _ImageNet: http://www.image-net.org/
+.. _alexcnwy: https://github.com/alexcnwy
diff --git a/bin/00resize.py b/bin/00resize.py
deleted file mode 100755
index dc1ba3d..0000000
--- a/bin/00resize.py
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/python3
-
-"""
-Python 3 
---------
-
-Only python3 has (finally!) a decent multiprocessing module, which
-handles KeyboardInterrupt (CTRL-C) *at all* and does this without need for
-any extra hassle and awkward try..except stunts. Thank you!
-
-verbose
--------
-
-Use -v to know which image is processed. With multiprocessing, the order is
-messed up a bit but you will still get a rough estimate of how many images are
-left, assuming all images are equal in size and thus all processes are
-approximately equally fast.
-
-multiprocessing vs. threads
----------------------------
-
-We use Python's multiprocessing instead of the default imagemagick OpenMP
-parallelization since the former is a little faster -- and well .. coding and
-benchmarking is fun!.
-
-wall clock times, dual-core box, using 
-  export OMP_NUM_THREADS=<max OpenMP threads>
-
-    multiprocessing, ncore = 1,2,4, max OpenMP threads = 1
-
-    $ for x in 1 2 4; do time ./00resize.py -n $x 0.2 files/*; done
-    
-    real    1m15.663s     # 1
-    real    0m38.577s     # 2  ***
-    real    0m39.365s     # 4
-    
-    multiprocessing, ncore = 1,2,4, max OpenMP threads = 2
-    
-    $ for x in 1 2 4; do time ./00resize.py -n $x 0.2 files/*; done
-    
-    real    0m46.304s     # 1  ***
-    real    0m38.766s     # 2
-    real    0m38.984s     # 4
-
-The figures to compare are marked with ***, i.e. 2 threads vs. 2 processes
-with 1 thread / core. With the latter, we are about factor 1.2 faster.
-
-Note: We know that even setting OMP_NUM_THREADS=1 (which is probably
-equivalent to "-limit threads 1" in the imagemagick case) is not a good idea
-since there is still overhead caused by the creation of an OpenMP thread. The
-only way to get rid of OpenMP completely is to re-compile imagemagick with
-./configure --disable-openmp .
-
-resize
-------
-"convert -resize" does some form of interpolation. The fastest resize (we only make
-the image smaller) is "convert -sample": simply delete rows and columns
-or, on other words *sample* only every Nth row and Mth column. This is 3x
-faster than "-resize". It introduces some aliasing artifacts, but we usually
-don't care as long as the main features are intact since we squash the image
-down to smth like 8x8 later anyway.
-"""
-
-import os, multiprocessing, subprocess, functools, argparse
-from imgcmp import cli, env
-
-def _worker(tup, fraction=None, tgtdir=None, nfiles=None, verbose=False):
-    idx, _src = tup
-    src = os.path.abspath(_src)
-    # /home/foo -> _home_foo -> home_foo
-    tgt = os.path.join(tgtdir, src.replace('/','_')[1:])
-    # -auto-orient: keep EXIF orientation information
-    cmd = "convert -limit thread 1 -sample {}% -auto-orient {} {}".format(
-            fraction*100, src, tgt)
-    if verbose >= 1:   
-        print("{} of {}".format(idx+1, nfiles))
-    if verbose >= 2:
-        print(cmd)
-    if not os.path.exists(tgt):
-        subprocess.call(cmd, shell=True)
-
-if __name__ == '__main__':
-    
-    desc = """
-Resize images to FRACTION*100 % with imagemagick's convert. Store them in dir TGTDIR
-with their full name with / replaced by _, such that /path/to/file.png becomes
-TGTDIR/path_to_file.png
-"""
-    parser = argparse.ArgumentParser(description=desc) 
-    parser.add_argument('fraction', metavar='FRACTION', type=float,
-                        help=r'fraction value for resizing (0..1), e.g. 0.25 '
-                             r'for 25 percent')
-    parser.add_argument('files', metavar='FILE', nargs='+',
-                        help='image file names')
-    parser.add_argument('-t', '--tgtdir',
-                        default=cli.convert_dr,
-                        help='store resized files here [%(default)s]')
-    parser.add_argument('-n', '--ncore',
-                        default=multiprocessing.cpu_count(), type=int,
-                        help='number of cores for parallel work [%(default)s]')
-    parser.add_argument('-v', '--verbose', default=0, action='count',
-                        help='increase verbosity level, -vv prints convert commands')
-    args = parser.parse_args()
-    worker = functools.partial(_worker, 
-                               fraction=args.fraction,
-                               tgtdir=args.tgtdir,
-                               nfiles=len(args.files),
-                               verbose=args.verbose)
-    pool = multiprocessing.Pool(args.ncore)
-    pool.map(worker, enumerate(args.files))
-
diff --git a/bin/10fingerprints.py b/bin/10fingerprints.py
deleted file mode 100755
index 66d738f..0000000
--- a/bin/10fingerprints.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python3
-
-import sys, multiprocessing, functools, argparse, os
-from PIL import Image
-from imgcmp import calc, io, cli, env
-import numpy as np
-pj = os.path.join
-
-
-def _worker(tup, size_x=None, fpsdct=None):
-    ii, name = tup     
-    img = Image.open(name)
-    fpsdct[ii] = calc.phash(img, 
-                            size=(size_x, size_x), 
-                            highfreq_factor=4, 
-                            backtransform=False).flatten()
-
-if __name__ == '__main__':
-    
-    desc = """
-Calculate fingerprint database.    
-"""
-    parser = argparse.ArgumentParser(description=desc) 
-    parser.add_argument('files', metavar='FILE', nargs='*',
-                        default=[pj(cli.convert_dr, x) for x in \
-                                 os.listdir(cli.convert_dr)],
-                        help='image file names, [default: '
-                             '{}/*]'.format(cli.convert_dr))
-    parser.add_argument('-x', dest='size_x',
-                        default=8, type=int,
-                        help='resize images to (size_x, size_x), fingerprints '
-                             'are then (size_x**2,) 1d arrays; large = more '
-                             'features, more strict similarity [%(default)s]')
-    parser.add_argument('-f', dest='dbfile',
-                        default=cli.dbfile,
-                        help='database HDF file [%(default)s]')
-    args = parser.parse_args()
-
-    # "parallel" dict for sharing between procs
-    manager = multiprocessing.Manager()
-    fpsdct = manager.dict() 
-
-    pool = multiprocessing.Pool(multiprocessing.cpu_count())
-    worker = functools.partial(_worker, 
-                               size_x=args.size_x,
-                               fpsdct=fpsdct)
-    pool.map(worker, enumerate(args.files))
-
-    # sort: order array to match file names in list `files`
-    fps = np.array([fpsdct[ii] for ii in np.sort(fpsdct.keys())])
-    # http://docs.h5py.org/en/latest/strings.html
-    files = np.array([f for f in args.files]).astype('S')
-    io.write_h5(args.dbfile, dict(files=files, fps=fps))    
-
diff --git a/bin/20cluster.py b/bin/20cluster.py
deleted file mode 100755
index a016780..0000000
--- a/bin/20cluster.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/python3
-
-import os, shutil, argparse
-from imgcmp import misc, io, calc, cli, env
-import numpy as np
-
-pj = os.path.join
-
-if __name__ == '__main__':
-    
-    desc = """
-Read fingerprints database, perform clustering.
-"""
-    parser = argparse.ArgumentParser(description=desc) 
-    parser.add_argument('sim', type=float, metavar='SIM',
-                        help='similarity (0..1) where e.g. 0.1 = strict '
-                             'similarity (very similar images + small '
-                             'clusters); try 0.2 first, then increase for more '
-                             'but less similar images per cluster')
-    parser.add_argument('-f', dest='dbfile',
-                        default=cli.dbfile,
-                        help='database HDF file [%(default)s]')
-    args = parser.parse_args()
-    
-    db = io.read_h5(args.dbfile)
-    files = db['/files'] 
-    fps = db['/fps']
-     
-    # handle numpy 'S' type strings from HDF file:
-    # convert array([b'x', b'y'], dtype='|S1') -> ['a', 'b']
-    if type(files[0]) in [type(b'x'), np.bytes_]:
-        files = [f.decode() for f in files]
-
-    # {1: [list_of_files], 2: [list_of_files], ...}
-    cluster_dct = calc.cluster(files, fps, args.sim, 'hamming')
-
-    # [[list_of_files], [list_of_files], ...]
-    clst_multi = [x for x in cluster_dct.values() if len(x) > 1]
-
-    # {number_of_files1: [[list_of_files], [list_of_files],...],
-    #  number_of_files2: [[list_of_files],...],
-    # }
-    cdct_multi = {}
-    for x in clst_multi:
-        nn = len(x)
-        if not (nn in cdct_multi.keys()):
-            cdct_multi[nn] = [x]
-        else:    
-            cdct_multi[nn].append(x)
-
-    print("cluster dir: {}".format(cli.cluster_dr))
-    print("items per cluster : number of such clusters")
-    shutil.rmtree(cli.cluster_dr)
-    for n_in_cluster in np.sort(list(cdct_multi.keys())):
-        cluster_list = cdct_multi[n_in_cluster]
-        print("{} : {}".format(n_in_cluster, len(cluster_list)))
-        for iclus, lst in enumerate(cluster_list):
-            dr = pj(cli.cluster_dr, 
-                    'cluster_with_{}'.format(n_in_cluster),
-                    'cluster_{}'.format(iclus))
-            for fn in lst:
-                link = pj(dr, os.path.basename(fn))
-                misc.makedirs(os.path.dirname(link))
-                os.symlink(os.path.abspath(fn), link)
diff --git a/imagecluster/__init__.py b/imagecluster/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/imgcmp/env.py b/imagecluster/env.py
similarity index 100%
rename from imgcmp/env.py
rename to imagecluster/env.py
diff --git a/imagecluster/imagecluster.py b/imagecluster/imagecluster.py
new file mode 100644
index 0000000..b5acc6e
--- /dev/null
+++ b/imagecluster/imagecluster.py
@@ -0,0 +1,146 @@
+from scipy.spatial import distance
+from scipy.cluster import hierarchy
+import numpy as np
+from matplotlib import pyplot as plt
+
+import PIL.Image, os, multiprocessing, shutil
+from keras.applications.vgg16 import VGG16
+from keras.preprocessing import image
+from keras.applications.vgg16 import preprocess_input
+from keras.models import Model
+
+pj = os.path.join
+
+def get_model():
+    # base_model.summary():
+    #     ....
+    #     block5_conv4 (Conv2D)        (None, 15, 15, 512)       2359808   
+    #     _________________________________________________________________
+    #     block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         
+    #     _________________________________________________________________
+    #     flatten (Flatten)            (None, 25088)             0         
+    #     _________________________________________________________________
+    #     fc1 (Dense)                  (None, 4096)              102764544 
+    #     _________________________________________________________________
+    #     fc2 (Dense)                  (None, 4096)              16781312  
+    #     _________________________________________________________________
+    #     predictions (Dense)          (None, 1000)              4097000   
+    # 
+    # model: get output from pre-last fully connected layer 'fc2'
+    base_model = VGG16(weights='imagenet', include_top=True)
+    model = Model(inputs=base_model.input,
+                  outputs=base_model.get_layer('fc2').output)
+    return model
+
+
+def fingerprint(fn, model, size):
+    # keras.preprocessing.image.load_img() uses img.rezize(shape) with the
+    # default interpolation which is pretty bad (see
+    # imagecluster/play/pil_resample_methods.py). Given that we are restricted
+    # to small inputs of 244x244 by the VGG network, we should do our best to
+    # keep as much information from the original image as possible. This is a
+    # gut feeling, untested. But given that model.predict() is 10x slower than
+    # the while PIL image loading and resizing .. who cares.
+    #
+    # (244, 244, 3)
+    ##img = image.load_img(fn, target_size=size)
+    img = PIL.Image.open(fn).resize(size, 2)
+    # (244, 244, 3)
+    arr3d = image.img_to_array(img)
+    # (1, 244, 244, 3)
+    arr4d = np.expand_dims(arr3d, axis=0)
+    # (1, 244, 244, 3)
+    arr4d_pp = preprocess_input(arr4d)
+    return model.predict(arr4d_pp)[0,:]
+
+
+def _worker(fn, model, size):
+    print(fn)
+    return fn, fingerprint(fn, model, size)
+
+
+def fingerprints(files, model, size=(224,224)):
+    # Cannot use multiprocessing:
+    # TypeError: can't pickle _thread.lock objects 
+    # The error doesn't come from functools.partial since those objects are
+    # pickable since python3. The reason is the keras.model.Model, which is not
+    # pickable. However keras with tensorflow backend runs multithreaded
+    # (model.predict()), so we don't need that.
+##    worker = functools.partial(_worker,
+##                               model=model,
+##                               size=size)
+##    pool = multiprocessing.Pool(multiprocessing.cpu_count())
+##    return dict(pool.map(worker, files))
+    return dict(_worker(fn, model, size) for fn in files)
+
+
+def make_links(clusters, cluster_dr):
+    # [[list_of_files], [list_of_files], ...]
+    clst_multi = [x for x in clusters.values() if len(x) > 1]
+
+    # {number_of_files1: [[list_of_files], [list_of_files],...],
+    #  number_of_files2: [[list_of_files],...],
+    # }
+    cdct_multi = {}
+    for x in clst_multi:
+        nn = len(x)
+        if not (nn in cdct_multi.keys()):
+            cdct_multi[nn] = [x]
+        else:
+            cdct_multi[nn].append(x)
+
+    print("cluster dir: {}".format(cluster_dr))
+    print("items per cluster : number of such clusters")
+    if os.path.exists(cluster_dr):
+        shutil.rmtree(cluster_dr)
+    for n_in_cluster in np.sort(list(cdct_multi.keys())):
+        cluster_list = cdct_multi[n_in_cluster]
+        print("{} : {}".format(n_in_cluster, len(cluster_list)))
+        for iclus, lst in enumerate(cluster_list):
+            dr = pj(cluster_dr,
+                    'cluster_with_{}'.format(n_in_cluster),
+                    'cluster_{}'.format(iclus))
+            for fn in lst:
+                link = pj(dr, os.path.basename(fn))
+                os.makedirs(os.path.dirname(link), exist_ok=True)
+                os.symlink(os.path.abspath(fn), link)
+
+
+def get_files(dr):
+    return [pj(dr,base) for base in os.listdir(dr)] 
+
+
+def cluster(files, fps, sim=0.5, method='average', metric='euclidean'):
+    """Hierarchical clustering of images `files` based on image fingerprints
+    `fps`.
+
+    files : list of file names
+    sim : float 0..1
+        similarity tolerance (1=max. allowed similarity tolerance, all images
+        are considered similar and are in one cluster, 0=zero similarity
+        allowed, each image is it's own cluster of size 1)
+    fps : 2d array (len(files), N)
+        flattened fingerprints (1d array for each image), where `size` is the
+        argument to one of the *hash() functions 
+    method : see scipy.hierarchy.linkage(), all except 'centroid' produce
+        pretty much the same result
+    metric : see scipy.hierarchy.linkage(), make sure to use 'euclidean' in
+        case of method='centroid', 'median' or 'ward'
+    """
+    dfps = distance.pdist(fps, metric)
+    # hierarchical/agglomerative clustering (Z = linkage matrix, construct
+    # dendrogram) 
+    Z = hierarchy.linkage(dfps, method=method, metric=metric)
+    # cut dendrogram, extract clusters
+    cut = hierarchy.fcluster(Z, t=dfps.max()*sim, criterion='distance')
+    clusters = dict((ii,[]) for ii in np.unique(cut))
+    for iimg,iclus in enumerate(cut):
+        clusters[iclus].append(files[iimg])
+    return clusters 
+
+
+def view_image_list(lst):
+    for filename in lst:
+        fig,ax = plt.subplots()
+        ax.imshow(plt.imread(filename))
+    plt.show()
diff --git a/imagecluster/main.py b/imagecluster/main.py
new file mode 100644
index 0000000..f7bce90
--- /dev/null
+++ b/imagecluster/main.py
@@ -0,0 +1,26 @@
+import os, pickle
+import numpy as np
+from imagecluster import imagecluster as ic
+pj = os.path.join
+
+def main(imagedir, sim=0.5):
+    """Example main app using this library.     """
+    dbfn = pj(imagedir, 'fingerprints.pk')
+    if not os.path.exists(dbfn):
+        print("no fingerprints database {} found".format(dbfn))
+        files = ic.get_files(imagedir)
+        model = ic.get_model()
+        print("running all images thru NN model ...".format(dbfn))
+        fps = ic.fingerprints(files, model, size=(224,224))
+        with open(dbfn, 'wb') as fd:
+            pickle.dump(fps, fd)
+        fd.close()
+    else:
+        print("loading fingerprints database {} ...".format(dbfn))
+        with open(dbfn, 'rb') as fd:
+            fps = pickle.load(fd)
+    print("clustering ...")
+    clusters = ic.cluster(list(fps.keys()), 
+                          np.array(list(fps.values())), 
+                          sim)
+    ic.make_links(clusters, pj(imagedir, 'clusters'))
diff --git a/imagecluster/test/test.py b/imagecluster/test/test.py
new file mode 100644
index 0000000..9dbb6f0
--- /dev/null
+++ b/imagecluster/test/test.py
@@ -0,0 +1,39 @@
+import numpy as np
+import scipy.ndimage as ni
+from scipy import misc
+import tempfile, shutil, os, pickle
+
+# https://stackoverflow.com/a/39708493
+import logging
+logging.getLogger("tensorflow").setLevel(logging.ERROR)
+
+from imagecluster import main
+pj = os.path.join
+
+
+def test():
+    """Basic test of main()."""
+    try:
+        filedir = tempfile.mkdtemp(prefix='imagecluster_')
+        arr = misc.face()
+        images = [arr, 
+                  ni.gaussian_filter(arr, 10), 
+                  ni.gaussian_filter(arr, 20)]
+        image_fns = []
+        for idx, arr in enumerate(images):
+            fn = pj(filedir, 'image_{}.png'.format(idx))
+            misc.imsave(fn, arr)
+            image_fns.append(fn)
+        # first run: create fingerprints database, run clustering
+        main.main(filedir)
+        # second run, only run clustering, should be much faster
+        main.main(filedir)
+        with open(pj(filedir, 'fingerprints.pk'), 'rb') as fd:
+            fps = pickle.load(fd)
+        assert len(fps.keys()) == 3
+        assert set(fps.keys()) == set(image_fns)
+        for kk,vv in fps.items():
+            assert isinstance(vv, np.ndarray)
+            assert len(vv) == 4096
+    finally:
+        shutil.rmtree(filedir)
diff --git a/imgcmp.egg-info/PKG-INFO b/imgcmp.egg-info/PKG-INFO
deleted file mode 100644
index 5cb839b..0000000
--- a/imgcmp.egg-info/PKG-INFO
+++ /dev/null
@@ -1,173 +0,0 @@
-Metadata-Version: 1.0
-Name: imgcmp
-Version: 0.0.0
-Summary: Cluster images.
-Home-page: https://github.com/elcorto/imgcmp
-Author: Steve Schmerler
-Author-email: git@elcorto.com
-License: GPLv3
-Description: About
-        =====
-        Package for comparing images by content. Uses simple image hashing
-        ("fingerprints"). See "Methods" below for details.
-        
-        Install
-        =======
-        
-        ::
-        
-            git clone ...
-            pip3 install -e .
-        
-        Usage
-        =====
-        
-        There are 3 scripts in ``bin/``::
-        
-            00resize.py  10fingerprints.py  20cluster.py
-        
-        These must be executed one after another. Have a look at the help (use
-        ``<script> -h``).
-        
-        The first one, resizing images, is optional. However, if you have large images
-        (several MB/image), you should really resize them first in order to lower the
-        computational cost of subsequent operations. This makes sense if you want to
-        play with various parameters of the clustering.
-        
-        ::
-        
-            $ ./bin/00resize.py 0.2 /path/to/large/pics/* -vv
-        
-        This will resize all images to 20% (factor 0.2) of their original size. Resized
-        files are written to ``~/.imgcmp/convert/`` by default. Now, calculate the
-        fingerprint database::
-        
-            $ ./bin/10fingerprints.py 
-        
-        This uses all files in ``~/.imgcmp/convert/*`` by default. Alternatively,
-        specify all images on the command line. It creates a file
-        ``~/.imgcmp/fingerprints.hdf`` (HDF5 file format, fingerprint database). 
-        
-        Last, we will cluster the images (actually their fingerprints) by using a
-        similarity index ( ``0..1``, ``0.3`` below). The index can be thought of as the
-        allowed *dissimilarity*. A small index means to put only very similar images in
-        one cluster. The extreme case 0.0 means to allow zero dissimilarity and thus
-        put each image in a cluster of size 1. In contrast, large values imply less
-        strict clustering and will put more but less similar images in a cluster. A
-        value of 1.0 is equal to putting all images in one single cluster (e.g. all
-        images are treated as equal)::
-        
-            $ ./bin/20cluster.py 0.3
-            cluster dir: /home/user/.imgcmp/cluster
-            items per cluster : number of such clusters
-            2 : 41
-            3 : 2
-            4 : 2
-        
-        By default, the database ``~/.imgcmp/fingerprints.hdf`` is used. Have a look at
-        the clusters (as dirs with symlinks to the relevant files)::
-        
-            $ ls ~/.imgcmp/cluster/
-            cluster_with_2  cluster_with_3  cluster_with_4
-        
-            $ ls ~/.imgcmp/cluster/cluster_with_2/
-            cluster_0   cluster_1  cluster_2
-        
-            $ qiv -ifm ~/.imgcmp/cluster/cluster_with_2/cluster_0/
-        
-        Methods
-        =======
-        
-        What we can do and what not
-        ---------------------------
-        
-        We use a variant of the "phash" method -- a simple and fast way to calculate
-        fingerprints. The package can detect images which are rather similar, e.g. the
-        same scene photographed twice or more with some camera movement in between, or
-        a scene with the same background and e.g. one person exchanged. Good parameter
-        values are x=8 ... 16 (``10fingerprints.py -x``) and similarity index sim=0.1
-        ... 0.3 (``20cluster.py <sim>``). However, values above sim=0.3 will quickly
-        recognize all sorts of images as equal and usually, results obtained with sim >
-        0.5 are seldom useful. Next, the higher x, the more detailed and close to the
-        original image the fingerprint is, since the original is squashed down to a x
-        times x grayscale image, converted to binary (pixed is black or white), and
-        flattened into a vector of x**2 length (the "fingerprint"). High values such as
-        64 already result in detection of only extremely similar images, since too much
-        information of the original image is kept. Instead, a value of x=2 is obviously
-        useless because it reduces each image to a 2x2 matrix (and fingerprint vector of
-        length 4) with almost no information from the original image left. So x=8 and
-        sim=0.1 is roughly equivalent to x=16 and sim=0.3. Therefore, if you get too
-        many "false positives" (images counted as similar when they are not), reduce
-        sim or increase x.
-        
-        The original goal was to have a clustering based on classification of image
-        *content* -- smth like: image A this an image of my kitchen; image B is also an
-        image of my kitchen, only from a different angle and some persons in the
-        foreground, but the information -- this is my kitchen -- is the same. This is a
-        feature-detection task which relies on the ability to recognize *objects*
-        within a scene, regardless of other scene parameters (like view angle, color,
-        light, ...). It turns out that we may need Neural Networks and some real
-        machine learning for the generation of better *feature vectors*, e.g. a feature
-        vector that always encodes the information "my kitchen" -- similar to DNNs
-        which learn features automatically. The simple image hashing done here is
-        rather limited in that respect. It only does a very pedestrian smoothing /
-        low-pass filtering to reduce the noise and extract the "important" parts of the
-        image. But this helps to find duplicates and almost-duplicates in a collection
-        of photos. 
-        
-        Note that we do not want to do the clustering with an NN, only the feature
-        vector generation. The clustering shall be done in a classical fashion as used
-        here (hierarchical/agglomerative clustering).
-        
-        NN-related keywords for doing it right:
-        
-        * distributed representation/embedding: the network has an abstract
-          representation of the object ("my kitchen"), distributed across the whole net
-          via the weights
-        * unsupervised learning: learn that all images should produce the same net
-          output "my kitchen" w/o labeled training data -- i.e. find clusters
-          (structure in the data) automatically
-        * neighorhood component analysis, wasabi, category learning
-        
-        image fingerprints: simple and fast
-        -----------------------------------
-        These methods basically squash down the image to something like 16x16,
-        transform to gray scale and store that as a feature vector of length 16x16, for
-        example -> fast. But the method is not invariant against rotation and the
-        like.
-        
-        The idea is always to calculate a database of image fingerprints ("hashes",
-        feature vectors) and then do searches in feature space (all fingerprints) using
-        some form of KD-tree / nearest neighbor search.
-        
-        * google: calculate image fingerprint
-        * [a|p|d]hash:
-          https://realpython.com/blog/python/fingerprinting-images-for-near-duplicate-detection/ 
-        * especially: phash.org
-        * older Perl implementation of a ahash(?)-like method:
-          http://www.jhnc.org/findimagedupes/manpage.html, also as Debian package
-        
-        more scientific "feature extraction"
-        ------------------------------------
-        
-        * classical CV (computer vision): SIFT (good but slow, old-school
-          hand-engineered feature detector), SURF (faster version of
-          SIFT)
-        * http://opencv-python-tutroals.readthedocs.org/en/latest/index.html
-            * SIFT and SURF are patented, so use ORB instead
-              http://opencv-python-tutroals.readthedocs.org/en/latest/py_tutorials/py_feature2d/py_orb/py_orb.html#orb
-        * opencv Bag Of Words: http://stackoverflow.com/questions/7205489/opencv-fingerprint-image-and-compare-against-database
-        
-        Python image processing
-        -----------------------
-        * google: python image processing :)
-        * http://scikit-image.org/
-        * PIL vs. Pillow: http://docs.python-guide.org/en/latest/scenarios/imaging/
-        * http://www.scipy-lectures.org/advanced/image_processing
-        
-        better methods
-        --------------
-        read about: Content-based image classification
-        
-Keywords: image cluster
-Platform: UNKNOWN
diff --git a/imgcmp.egg-info/SOURCES.txt b/imgcmp.egg-info/SOURCES.txt
deleted file mode 100644
index 072c258..0000000
--- a/imgcmp.egg-info/SOURCES.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-README.rst
-bin/00resize.py
-bin/10fingerprints.py
-bin/20cluster.py
-imgcmp/__init__.py
-imgcmp/calc.py
-imgcmp/cli.py
-imgcmp/env.py
-imgcmp/io.py
-imgcmp/misc.py
-imgcmp.egg-info/PKG-INFO
-imgcmp.egg-info/SOURCES.txt
-imgcmp.egg-info/dependency_links.txt
-imgcmp.egg-info/top_level.txt
\ No newline at end of file
diff --git a/imgcmp.egg-info/dependency_links.txt b/imgcmp.egg-info/dependency_links.txt
deleted file mode 100644
index 8b13789..0000000
--- a/imgcmp.egg-info/dependency_links.txt
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/imgcmp.egg-info/top_level.txt b/imgcmp.egg-info/top_level.txt
deleted file mode 100644
index 07b989c..0000000
--- a/imgcmp.egg-info/top_level.txt
+++ /dev/null
@@ -1 +0,0 @@
-imgcmp
diff --git a/imgcmp/__init__.py b/imgcmp/__init__.py
deleted file mode 100644
index a894a7f..0000000
--- a/imgcmp/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__all__ = ['calc', 'misc', 'io']
diff --git a/imgcmp/calc.py b/imgcmp/calc.py
deleted file mode 100644
index 0160344..0000000
--- a/imgcmp/calc.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# https://github.com/JohannesBuchner/imagehash
-# http://www.hackerfactor.com/blog/index.php?/archives/432-Looks-Like-It.html
-
-from imgcmp import env
-import numpy as np
-import scipy.fftpack as fftpack
-from scipy.spatial import distance
-from scipy.cluster import hierarchy
-
-INT = np.int32
-FLOAT = np.float64
-
-
-def img2arr(img, size=(8,8), dtype=INT, resample=2):
-    """Convert PIL Image to gray scale and resample to numpy array of shape
-    `size` and `dtype`.
-
-    Parameters
-    ----------
-    img : PIL Image
-    size : (int, int)
-        size of square fingerprint array for `img`
-    resample : int
-        interpolation method, see help of ``PIL.Image.Image.resize``
-    """
-    # convert('L'): to 1D grey scale array
-    return np.array(img.convert("L").resize(size, resample), dtype=dtype)
-
-
-def ahash(img, size=(8,8)):
-    """
-    Parameters
-    ----------
-    img : PIL image
-    size : (int, int)
-        size of square fingerprint array for `img`
-    """
-    pixels = img2arr(img, size=size)
-    return (pixels > pixels.mean()).astype(INT)
-
-
-def phash(img, size=(8,8), highfreq_factor=4, backtransform=False):
-    img_size = (size[0]*highfreq_factor, 
-                size[1]*highfreq_factor)
-    pixels = img2arr(img, size=img_size, dtype=np.float64)
-    fpixels = fftpack.dct(fftpack.dct(pixels, axis=0), axis=1)
-    # XXX we had fpixels[1:size[0], 1:size[1]] before, find out why
-    fpixels_lowfreq = fpixels[:size[0], :size[1]]
-    if backtransform:
-        tmp = fftpack.idct(fftpack.idct(fpixels_lowfreq, axis=0), axis=1)
-    else:
-        tmp = fpixels_lowfreq
-    return (tmp > np.median(tmp)).astype(INT)
-
-
-def dhash(img, size=(8,8)):
-    pixels = img2arr(img, size=(size[0] + 1, size[1]))
-    return (pixels[1:, :] > pixels[:-1, :]).astype(INT)
-
-
-def cluster(files, fps, sim=0.2, metric='hamming'):
-    """Hierarchical clustering of images `files` based on image fingerprints
-    `fps`.
-
-    files : list of file names
-    sim : float
-        similarity (1=max. allowed similarity, all images are considered
-        similar and are in one cluster, 0=zero similarity allowed, each image
-        is it's own cluster of size 1)
-    fps : 2d array (len(files), size[0]*size[1])
-        flattened fingerprints (1d array for each image), where `size` is the
-        argument to one of the *hash() functions 
-    """
-    dfps = distance.pdist(fps.astype(bool), metric)
-    # hierarchical/agglomerative clustering (Z = linkage matrix, construct
-    # dendrogram) 
-    Z = hierarchy.linkage(dfps, method='average', metric=metric)
-    # cut dendrogram, extract clusters
-    cut = hierarchy.fcluster(Z, t=dfps.max()*sim, criterion='distance')
-    clusters = dict((ii,[]) for ii in np.unique(cut))
-    for iimg,iclus in enumerate(cut):
-        clusters[iclus].append(files[iimg])
-    return clusters 
diff --git a/imgcmp/cli.py b/imgcmp/cli.py
deleted file mode 100644
index ce7d74e..0000000
--- a/imgcmp/cli.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import os
-from imgcmp import misc, env
-pj = os.path.join
-
-base_dir = pj(os.environ['HOME'], '.imgcmp')
-convert_dr = pj(base_dir, 'convert')
-cluster_dr = pj(base_dir, 'cluster')
-dbfile = pj(base_dir, 'fingerprints.hdf')
-
-for pp in base_dir, convert_dr, cluster_dr:
-    misc.makedirs(pp)
diff --git a/imgcmp/io.py b/imgcmp/io.py
deleted file mode 100644
index 539a775..0000000
--- a/imgcmp/io.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from imgcmp import env
-import h5py
-
-def write_h5(fn, dct, mode='w', **kwds):
-    fh = h5py.File(fn, mode=mode, **kwds)
-    for key,val in dct.items():
-        _key = key if key.startswith('/') else '/'+key
-        fh[_key] = val
-    fh.close()
-
-
-def read_h5(fn):
-    fh = h5py.File(fn, mode='r')
-    dct = {}
-    def get(key, obj, dct=dct):
-        if isinstance(obj, h5py.Dataset):
-            _key = key if key.startswith('/') else '/'+key
-            dct[_key] = obj.value
-    fh.visititems(get)
-    fh.close()
-    return dct
-
-
-def read_db(dbfile):
-    db = read_h5(dbfile)
-    return db['/files'], db['/fps']
diff --git a/imgcmp/misc.py b/imgcmp/misc.py
deleted file mode 100644
index d187352..0000000
--- a/imgcmp/misc.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import itertools, math, os
-
-def makedirs(path):
-    """Same as os.makedirs() but silently skips existing dirs."""
-    pp = './' if path.strip() == '' else path
-    if not os.path.exists(pp):
-        os.makedirs(pp)
-
-
-# http://stackoverflow.com/a/25649669
-def idx_square2cond(ii, jj, nn):
-    return ii*nn + jj - ii*(ii+1)/2 - ii - 1
-
-# http://stackoverflow.com/a/14839010
-def idx_cond2square(kk, nn):
-    b = 1 - 2*nn
-    x = int(math.floor((-b - math.sqrt(b**2 - 8*kk))/2))
-    y = kk + x*(b + x + 2)/2 + 1
-    return (x,y)  
-
-
-def view_lst(lst):
-    from pwtools.common import system
-    cmd = 'qiv -fm ' + ' '.join(lst)
-    system(cmd, wait=True)
diff --git a/imgcmp/tests/test_idx.py b/imgcmp/tests/test_idx.py
deleted file mode 100644
index 6ce45eb..0000000
--- a/imgcmp/tests/test_idx.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import numpy as np
-from scipy.spatial import distance
-import itertools
-from imgcmp import misc
-
-def test_pdist():
-    # play with pdist, which returns the upper triangle of the full squareform
-    # distance matrix, test our indexing routines (thank you
-    # http://stackoverflow.com)
-    nvec = 100
-    dim = 10
-    triu_list = zip(*np.triu_indices(nvec, k=1))
-    vecs = np.random.rand(nvec, dim)
-    dist1d = distance.pdist(vecs)
-    assert len(triu_list) == len(dist1d)
-    dist2d = distance.squareform(dist1d)
-    for ii,jj in itertools.product(range(nvec), repeat=2):
-        if ii < jj:
-            idx_1d = misc.idx_square2cond(ii, jj, nvec)
-            assert dist1d[idx_1d] == dist2d[ii,jj]
-            assert misc.idx_cond2square(idx_1d, nvec) == (ii,jj)
-            assert triu_list[idx_1d] == (ii,jj)
diff --git a/setup.py b/setup.py
index 757bd0c..385aa5f 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 # publish on pypi
 # ---------------
 #   $ python3 setup.py sdist
-#   $ twine upload dist/imgcmp-x.y.z.tar.gz 
+#   $ twine upload dist/imagecluster-x.y.z.tar.gz 
 
 import os, importlib
 from setuptools import setup
@@ -21,12 +21,12 @@
 # I don't get it, seriously.
 install_requires = []
 req = [('numpy', 'numpy', None, None),
-       ('docopt', 'docopt', None, None),
        ('tensorflow', 'tensorflow', None, None),
        ('keras', 'keras', None, None),
        ]
 
 for pipname,pkgname,op,ver in req:
+    print("checking dependency: {}".format(pkgname))
     if op and ver:
         this_req = pipname + op + ver
     else:
@@ -42,16 +42,17 @@
         install_requires.append(this_req)
 
 setup(
-    name='imgcmp',
+    name='imagecluster',
     version='0.0.0',
-    description='Cluster images.',
+    description='cluster images based on image content using a pre-trained ' \
+'deep neural network and hierarchical clustering',
     long_description=long_description,
-    url='https://github.com/elcorto/imgcmp',
+    url='https://github.com/elcorto/imagecluster',
     author='Steve Schmerler',
     author_email='git@elcorto.com',
     license='GPLv3',
-    keywords='image cluster',
-    packages=['imgcmp'],
+    keywords='image cluster vgg16 deep-learning',
+    packages=['imagecluster'],
     install_requires=install_requires,
-    scripts=['{}/{}'.format(bindir, script) for script in os.listdir(bindir)]
+##    scripts=['{}/{}'.format(bindir, script) for script in os.listdir(bindir)]
 )