Improve support for binary assets

src/webassets/filter/__init__.py

@@ -492,7 +492,7 @@ def created(self):
 
         try:
             data = (data.read() if hasattr(data, 'read') else data)
-            if data is not None:
+            if isinstance(data, unicode):
                 data = data.encode('utf-8')
 
             if input_file.created:

src/webassets/filter/sass.py

@@ -125,7 +125,7 @@ def _apply_sass(self, _in, out, cd=None):
                                     # shell: necessary on windows to execute
                                     # ruby files, but doesn't work on linux.
                                     shell=(os.name == 'nt'))
-            stdout, stderr = proc.communicate(_in.read().encode('utf-8'))
+            stdout, stderr = proc.communicate(_in.read())
 
             if proc.returncode != 0:
                 raise FilterError(('sass: subprocess had error: stderr=%s, '+
@@ -134,7 +134,7 @@ def _apply_sass(self, _in, out, cd=None):
             elif stderr:
                 print("sass filter has warnings:", stderr)
 
-            out.write(stdout.decode('utf-8'))
+            out.write(stdout)
         finally:
             if cd:
                 os.chdir(old_dir)

src/webassets/merge.py

@@ -10,7 +10,7 @@
     from urllib2 import Request as URLRequest, urlopen
     from urllib2 import HTTPError
 import logging
-from io import open
+from io import open, BytesIO
 from webassets.six.moves import filter
 
 from .utils import cmp_debug_levels, StringIO, hash_func
@@ -56,8 +56,11 @@ def data(self):
         raise NotImplementedError()
 
     def save(self, filename):
-        with open(filename, 'w') as f:
-            f.write(self.data())
+        data = self.data()
+        if isinstance(data, unicode):
+            data = data.encode('utf-8')
+        with open(filename, 'wb') as f:
+            f.write(data)
 
 
 class FileHunk(BaseHunk):
@@ -74,7 +77,7 @@ def mtime(self):
         pass
 
     def data(self):
-        f = open(self.filename, 'r', encoding='utf-8')
+        f = open(self.filename, 'rb')
         try:
             return f.read()
         finally:
@@ -159,11 +162,11 @@ def data(self):
         return self._data
 
     def save(self, filename):
-        f = open(filename, 'w', encoding='utf-8')
-        try:
-            f.write(self.data())
-        finally:
-            f.close()
+        data = self.data()
+        if isinstance(data, unicode):
+            data = data.encode('utf-8')
+        with open(filename, 'wb') as f:
+            f.write(data)
 
 
 def merge(hunks, separator=None):
@@ -222,6 +225,32 @@ def _wrap_cache(self, key, func):
             self.cache.set(key, content)
         return MemoryHunk(content)
 
+    def create_input_buffer_for(self, data):
+        have_binary = isinstance(data, BytesIO)
+
+        if isinstance(data, unicode):
+            return StringIO(data)
+        else:
+            return BytesIO(data)
+
+    def convert_input_buffer_for(self, filter, data):
+        accepts_binary = getattr(filter, 'binary_input', False)
+        have_binary = isinstance(data, BytesIO)
+
+        if accepts_binary and not have_binary:
+            return BytesIO(data.getvalue().encode('utf-8'))
+        elif not accepts_binary and have_binary:
+            return StringIO(data.getvalue().decode('utf-8'))
+        else:
+            return data
+
+    def create_output_buffer_for(self, filter):
+        if getattr(filter, 'binary_output', False):
+            return BytesIO(b'')
+        else:
+            # For 2.x, StringIO().getvalue() returns str
+            return StringIO(u'')
+
     def apply(self, hunk, filters, type, kwargs=None):
         """Apply the given list of filters to the hunk, returning a new
         ``MemoryHunk`` object.
@@ -244,11 +273,12 @@ def apply(self, hunk, filters, type, kwargs=None):
         kwargs_final.update(kwargs or {})
 
         def func():
-            data = StringIO(hunk.data())
+            data = self.create_input_buffer_for(hunk.data())
             for filter in filters:
+                data = self.convert_input_buffer_for(filter, data)
+                out = self.create_output_buffer_for(filter)
                 log.debug('Running method "%s" of  %s with kwargs=%s',
                     type, filter, kwargs_final)
-                out = StringIO(u'') # For 2.x, StringIO().getvalue() returns str
                 getattr(filter, type)(data, out, **kwargs_final)
                 data = out
                 data.seek(0)

src/webassets/version.py

@@ -6,8 +6,8 @@
 
 import os
 import pickle
-from webassets import six
 
+from webassets import six
 from webassets.merge import FileHunk
 from webassets.utils import md5_constructor, RegistryMetaclass, is_url
 
@@ -162,8 +162,12 @@ def determine_version(self, bundle, ctx, hunk=None):
                 raise VersionIndeterminableError(
                     'output target has a placeholder')
 
+        data = hunk.data()
+        if isinstance(data, unicode):
+            data = data.encode('utf-8')
+
         hasher = self.hasher()
-        hasher.update(hunk.data().encode('utf-8'))
+        hasher.update(data)
         return hasher.hexdigest()[:self.length]
 
 

tests/test_hunks.py

@@ -0,0 +1,58 @@
+"""Test the versioners and manifest implementations.
+"""
+import hashlib
+import codecs
+import os
+
+from nose.tools import assert_raises
+
+from webassets.env import Environment
+from webassets.merge import MemoryHunk, FileHunk
+from webassets.test import TempEnvironmentHelper
+from webassets.version import (
+    FileManifest, JsonManifest, CacheManifest, TimestampVersion,
+    VersionIndeterminableError, HashVersion, get_versioner, get_manifest)
+
+
+class TestBinaryHunks(TempEnvironmentHelper):
+    BINARY_DATA = b'\x80\x90\xa0\xff'
+
+    def setup(self):
+        super(TestBinaryHunks, self).setup()
+        self.bundle = self.mkbundle('in', depends=('dep'), output='out')
+
+        self.name = 'in'
+        dirs = os.path.dirname(self.path(self.name))
+        if not os.path.exists(dirs):
+            os.makedirs(dirs)
+        f = codecs.open(self.path(self.name), 'wb')
+        f.write(self.BINARY_DATA)
+        f.close()
+
+    def test_read_binary_file_hunk(self):
+        h = FileHunk(self.path(self.name))
+        d = h.data() 
+        assert isinstance(d, str)
+        assert d == self.BINARY_DATA
+
+    def test_write_binary_file_hunk(self):
+        outfile = self.path('out')
+        h = FileHunk(self.path(self.name))
+        h.save(outfile)
+
+        with open(outfile, 'rb') as f:
+            d = f.read()
+
+        assert isinstance(d, str)
+        assert d == self.BINARY_DATA
+
+    def test_write_memory_hunk(self):
+        outfile = self.path('out')
+        h = MemoryHunk(self.BINARY_DATA)
+        h.save(outfile)
+
+        with open(outfile, 'rb') as f:
+            d = f.read()
+
+        assert isinstance(d, str)
+        assert d == self.BINARY_DATA