Merge pull request #10 from Kautenja/ck3

Polish Implementation

Makefile

@@ -10,6 +10,8 @@ BUILD=build
 PDFLATEX=pdflatex
 # the pointer to the bibilography engine (bibtex)
 BIBTEX=bibtex
+# the number of frames of interpolation to use in videos
+INTER_FRAMES=3
 
 # install Python dependencies in the requirements.txt
 install:
@@ -52,7 +54,7 @@ review_w_ref: build
 # Args:
 #   1: the name of the directory in the build directory to find frames in
 define frames_to_video
-	${PYTHON} frames_to_video.py build/$(1) build/$(1).mp4
+	${PYTHON} frames_to_video.py build/$(1) build/$(1).mp4 ${INTER_FRAMES}
 endef
 
 # make all the content reconstruction videos
@@ -61,9 +63,8 @@ content_videos:
 	$(call frames_to_video,content/block2_conv1)
 	$(call frames_to_video,content/block3_conv1)
 	$(call frames_to_video,content/block4_conv1)
-	$(call frames_to_video,content/block4_conv2/sgd)
-	$(call frames_to_video,content/block4_conv2/lbfgs)
 	$(call frames_to_video,content/block5_conv1)
+	$(call frames_to_video,content/block4_conv2)
 
 # make all the style reconstruction videos
 style_videos:
@@ -72,23 +73,17 @@ style_videos:
 	$(call frames_to_video,style/block3_conv1)
 	$(call frames_to_video,style/block4_conv1)
 	$(call frames_to_video,style/block5_conv1)
-	$(call frames_to_video,style/starry_night/sgd)
-	$(call frames_to_video,style/starry_night/lbfgs)
 
 # make all the style transfer videos
 transfer_videos:
-	$(call frames_to_video,transfer/kandinsky/sgd)
-	$(call frames_to_video,transfer/kandinsky/lbfgs)
-	$(call frames_to_video,transfer/monet/sgd)
-	$(call frames_to_video,transfer/monet/lbfgs)
-	$(call frames_to_video,transfer/scream/sgd)
-	$(call frames_to_video,transfer/scream/lbfgs)
-	$(call frames_to_video,transfer/seated-nudes/sgd)
-	$(call frames_to_video,transfer/seated-nudes/lbfgs)
-	$(call frames_to_video,transfer/shipwreck/sgd)
-	$(call frames_to_video,transfer/shipwreck/lbfgs)
-	$(call frames_to_video,transfer/starry-night/sgd)
-	$(call frames_to_video,transfer/starry-night/lbfgs)
+	$(call frames_to_video,transfer/seated-nude)
+	$(call frames_to_video,transfer/the-starry-night)
+	$(call frames_to_video,transfer/the-scream)
+	$(call frames_to_video,transfer/the-shipwreck-of-the-minotaur)
+	$(call frames_to_video,transfer/composition-vii)
+	$(call frames_to_video,transfer/houses-of-parliament/tv-0)
+	$(call frames_to_video,transfer/houses-of-parliament/tv-1e0)
+	$(call frames_to_video,transfer/houses-of-parliament/tv-1e1)
 
 # make all the built videos
 videos: content_videos style_videos transfer_videos

README.md

@@ -1,50 +1,90 @@
 # neural stylization
 
-An **implementation** of the paper
-[_A Neural Algorithm of Artistic Style_](https://arxiv.org/abs/1508.06576).
+An **implementation** of the arXiv preprint
+[_A Neural Algorithm of Artistic Style [1]_](#references)
+& paper 
+[_Image Style Transfer Using Convolutional Neural Networks [2]_](#references).
 
-![Monet](tex/img/transfer/monet.png)
+## Original Photograph: _Tubingen, Germany_
 
-![Scream](tex/img/transfer/scream.png)
+<p float="left" align="center">
+<img src="img/content/tubingen.jpg" width="375"/> 
+</p>
 
-![Seated Nudes](tex/img/transfer/seated-nudes.png)
+-----
 
-![Starry Starry Night](tex/img/transfer/starry-starry-night.png)
+## Claude Monet's _Houses of Parliament_
 
-![Shipwreck](tex/img/transfer/shipwreck.png)
+<p float="left" align="center">
+<img src="img/styles/houses-of-parliament.jpg" width="375"/>  
+<img src="tex/img/transfer/houses-of-parliament-tv-1e0.png" width="425"/> 
+</p>
 
-![Kandinsky](tex/img/transfer/kandinsky.png)
+## Pablo Picasso's _Seated Nude_
 
-## Getting Started
+<p float="left" align="center">
+<img src="img/styles/seated-nude.jpg" width="245"/>  
+<img src="tex/img/transfer/seated-nude.png" width="425"/> 
+</p>
+
+## Edvard Munch's _The Scream_
+
+<p float="left" align="center">
+<img src="img/styles/the-scream.jpg" width="250"/>  
+<img src="tex/img/transfer/the-scream.png" width="425"/> 
+</p>
+
+## Vincent van Gogh's _The Starry Night_
+
+<p float="left" align="center">
+<img src="img/styles/the-starry-night.jpg" width="400"/>  
+<img src="tex/img/transfer/the-starry-night.png" width="425"/> 
+</p>
+
+## William Turner's _The Shipwreck of The Minotaur_
+
+<p float="left" align="center">
+<img src="img/styles/the-shipwreck-of-the-minotaur.jpg" width="425"/>  
+<img src="tex/img/transfer/the-shipwreck-of-the-minotaur.png" width="400"/> 
+</p>
+
+## Wassily Kandinsky's _Composition VII_
+
+<p float="left" align="center">
+<img src="img/styles/composition-vii.jpg" width="425"/>  
+<img src="tex/img/transfer/composition-vii.png" width="425"/> 
+</p>
+
+# Getting Started
 
 These instructions will get you a copy of the project up and running on your
 local machine for development and testing purposes. See deployment for notes
 on how to deploy the project on a live system.
 
-### Prerequisites
+## Prerequisites
 
 1.  [python3][] - Programming Environment
 1.  [pip3][] - Python Dependency Management
 
 [python3]: https://python.org
 [pip3]: https://packaging.python.org/tutorials/installing-packages/
 
-### Installing
+## Installing
 
 To install dependent modules:
 
 ```shell
 pip3 install -r requirements.txt
 ```
 
-## Project Components
+# Project Components
 
 [neural_stylization][] contains Python modules with utility methods and
 classes for the project.
 
 [neural_stylization]: neural_stylization
 
-### VGG19
+## VGG19
 
 This project relies on the VGG19 architecture.
 [VGG19-classification.ipynb][] outlines some basic image classification
@@ -57,7 +97,7 @@ and normalizing image can be found in [neural_stylization/img_util.py][].
 [neural_stylization/vgg19.py]: neural_stylization/vgg19.py
 [neural_stylization/img_util.py]: neural_stylization/img_util.py
 
-### Content Reconstruction
+## Content Reconstruction
 
 [content-reconstruction.ipynb][] describes the content reconstruction
 process from white noise. Performing gradient descent of the content loss
@@ -66,7 +106,7 @@ representation of the networks activation for a given layer _l_.
 
 [content-reconstruction.ipynb]: content-reconstruction.ipynb
 
-### Style Reconstruction
+## Style Reconstruction
 
 [style-reconstruction.ipynb][] describes the style reconstruction
 process from white noise. Performing gradient descent of the style loss
@@ -75,7 +115,7 @@ representation of the networks activation for a given set of layers _L_.
 
 [style-reconstruction.ipynb]: style-reconstruction.ipynb
 
-### Style Transfer
+## Style Transfer
 
 [style-transfer.ipynb][] describes the style transfer process between a white
 noise image **x**, a content image **p**, and a style representation **a**.
@@ -88,8 +128,21 @@ and colors from **a**.
 # Acknowledgments
 
 -   [keras-team](https://github.com/keras-team) provides `Keras`, a high
-	level neural network framework. They also provide the pre-trained
-	ImageNet weights and some tutorials that help build this project.
+    level neural network framework. They also provide the pre-trained
+    ImageNet weights and some tutorials that help build this project.
+
+# References
+
+[_[1] L. A. Gatys, A. S. Ecker, and M. Bethge. A neural algorithm of artistic style. arXiv preprint
+arXiv:1508.06576, 2015._][ref1]
+
+[ref1]: https://arxiv.org/abs/1508.06576
+
+[_[2] L. A. Gatys, A. S. Ecker, and M. Bethge. Image style transfer using convolutional neural networks. In
+Computer Vision and Pattern Recognition (CVPR), 2016 IEEE Conference on, pages 2414–2423.
+IEEE, 2016._][ref2]
+
+[ref2]: https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf
 
 # How Should I Cite `neural-stylization`?
 

frames_to_video.py

@@ -5,9 +5,30 @@
 from os import listdir
 
 
-# get the directory from the command line input
+def pairs(iterable):
+    """
+    Return a new iterable over sequential pairs in the given iterable.
+    i.e. (0,1), (1,2), ..., (n-2,n-1)
+    Args:
+        iterable: the iterable to iterate over the pairs of
+    Returns: a new iterator over the pairs of the given iterator
+    """
+    # lazily import tee from `itertools`
+    from itertools import tee
+    # split the iterator into 2 identical iterators
+    a, b = tee(iterable)
+    # retrieve the next item from the b iterator
+    next(b, None)
+    # zip up the iterators of current and next items
+    return zip(a, b)
+
+
+# the directory of png files to make into a video
 directory = argv[1]
+# the name of the video file to write
 video_name = argv[2]
+# the number of frames to use for interpolation
+interpolation_frames = int(argv[3])
 
 # get the frames from the given directory in sorted order
 frames = [filename for filename in listdir(directory) if '.png' in filename]
@@ -23,8 +44,19 @@
 out = cv2.VideoWriter(video_name, fourcc, 20.0, (width, height))
 
 # write the frames to the video file
-for frame in frames:
-	out.write(frame)
+for frame, next_frame in pairs(frames):
+    # write the current frame
+    out.write(frame)
+    # iterate over the number of interpolation frames
+    for i in range(1, interpolation_frames + 1):
+        # calculate a frame between current and next frames
+        w = i / interpolation_frames
+        mid_frame = cv2.addWeighted(frame, 1 - w, next_frame, w, 0)
+        # write this mid frame to the video
+        out.write(mid_frame)
+
+# the last frame still needs written
+out.write(frames[-1])
 
 # cleanup and release the video
 out.release()