diff --git a/src/webassets/filter/__init__.py b/src/webassets/filter/__init__.py index 05767c7b..877ee894 100644 --- a/src/webassets/filter/__init__.py +++ b/src/webassets/filter/__init__.py @@ -492,7 +492,7 @@ def created(self): try: data = (data.read() if hasattr(data, 'read') else data) - if data is not None: + if isinstance(data, unicode): data = data.encode('utf-8') if input_file.created: diff --git a/src/webassets/filter/sass.py b/src/webassets/filter/sass.py index 03911a77..bd4f4b1a 100644 --- a/src/webassets/filter/sass.py +++ b/src/webassets/filter/sass.py @@ -125,7 +125,7 @@ def _apply_sass(self, _in, out, cd=None): # shell: necessary on windows to execute # ruby files, but doesn't work on linux. shell=(os.name == 'nt')) - stdout, stderr = proc.communicate(_in.read().encode('utf-8')) + stdout, stderr = proc.communicate(_in.read()) if proc.returncode != 0: raise FilterError(('sass: subprocess had error: stderr=%s, '+ @@ -134,7 +134,7 @@ def _apply_sass(self, _in, out, cd=None): elif stderr: print("sass filter has warnings:", stderr) - out.write(stdout.decode('utf-8')) + out.write(stdout) finally: if cd: os.chdir(old_dir) diff --git a/src/webassets/merge.py b/src/webassets/merge.py index 646663ad..a290b9bc 100644 --- a/src/webassets/merge.py +++ b/src/webassets/merge.py @@ -10,7 +10,7 @@ from urllib2 import Request as URLRequest, urlopen from urllib2 import HTTPError import logging -from io import open +from io import open, BytesIO from webassets.six.moves import filter from .utils import cmp_debug_levels, StringIO, hash_func @@ -56,8 +56,11 @@ def data(self): raise NotImplementedError() def save(self, filename): - with open(filename, 'w') as f: - f.write(self.data()) + data = self.data() + if isinstance(data, unicode): + data = data.encode('utf-8') + with open(filename, 'wb') as f: + f.write(data) class FileHunk(BaseHunk): @@ -74,7 +77,7 @@ def mtime(self): pass def data(self): - f = open(self.filename, 'r', encoding='utf-8') + f = open(self.filename, 'rb') try: return f.read() finally: @@ -159,11 +162,11 @@ def data(self): return self._data def save(self, filename): - f = open(filename, 'w', encoding='utf-8') - try: - f.write(self.data()) - finally: - f.close() + data = self.data() + if isinstance(data, unicode): + data = data.encode('utf-8') + with open(filename, 'wb') as f: + f.write(data) def merge(hunks, separator=None): @@ -222,6 +225,32 @@ def _wrap_cache(self, key, func): self.cache.set(key, content) return MemoryHunk(content) + def create_input_buffer_for(self, data): + have_binary = isinstance(data, BytesIO) + + if isinstance(data, unicode): + return StringIO(data) + else: + return BytesIO(data) + + def convert_input_buffer_for(self, filter, data): + accepts_binary = getattr(filter, 'binary_input', False) + have_binary = isinstance(data, BytesIO) + + if accepts_binary and not have_binary: + return BytesIO(data.getvalue().encode('utf-8')) + elif not accepts_binary and have_binary: + return StringIO(data.getvalue().decode('utf-8')) + else: + return data + + def create_output_buffer_for(self, filter): + if getattr(filter, 'binary_output', False): + return BytesIO(b'') + else: + # For 2.x, StringIO().getvalue() returns str + return StringIO(u'') + def apply(self, hunk, filters, type, kwargs=None): """Apply the given list of filters to the hunk, returning a new ``MemoryHunk`` object. @@ -244,11 +273,12 @@ def apply(self, hunk, filters, type, kwargs=None): kwargs_final.update(kwargs or {}) def func(): - data = StringIO(hunk.data()) + data = self.create_input_buffer_for(hunk.data()) for filter in filters: + data = self.convert_input_buffer_for(filter, data) + out = self.create_output_buffer_for(filter) log.debug('Running method "%s" of %s with kwargs=%s', type, filter, kwargs_final) - out = StringIO(u'') # For 2.x, StringIO().getvalue() returns str getattr(filter, type)(data, out, **kwargs_final) data = out data.seek(0) diff --git a/src/webassets/version.py b/src/webassets/version.py index 5bb71607..90f58d96 100644 --- a/src/webassets/version.py +++ b/src/webassets/version.py @@ -6,8 +6,8 @@ import os import pickle -from webassets import six +from webassets import six from webassets.merge import FileHunk from webassets.utils import md5_constructor, RegistryMetaclass, is_url @@ -162,8 +162,12 @@ def determine_version(self, bundle, ctx, hunk=None): raise VersionIndeterminableError( 'output target has a placeholder') + data = hunk.data() + if isinstance(data, unicode): + data = data.encode('utf-8') + hasher = self.hasher() - hasher.update(hunk.data().encode('utf-8')) + hasher.update(data) return hasher.hexdigest()[:self.length] diff --git a/tests/test_hunks.py b/tests/test_hunks.py new file mode 100644 index 00000000..076ca3c7 --- /dev/null +++ b/tests/test_hunks.py @@ -0,0 +1,58 @@ +"""Test the versioners and manifest implementations. +""" +import hashlib +import codecs +import os + +from nose.tools import assert_raises + +from webassets.env import Environment +from webassets.merge import MemoryHunk, FileHunk +from webassets.test import TempEnvironmentHelper +from webassets.version import ( + FileManifest, JsonManifest, CacheManifest, TimestampVersion, + VersionIndeterminableError, HashVersion, get_versioner, get_manifest) + + +class TestBinaryHunks(TempEnvironmentHelper): + BINARY_DATA = b'\x80\x90\xa0\xff' + + def setup(self): + super(TestBinaryHunks, self).setup() + self.bundle = self.mkbundle('in', depends=('dep'), output='out') + + self.name = 'in' + dirs = os.path.dirname(self.path(self.name)) + if not os.path.exists(dirs): + os.makedirs(dirs) + f = codecs.open(self.path(self.name), 'wb') + f.write(self.BINARY_DATA) + f.close() + + def test_read_binary_file_hunk(self): + h = FileHunk(self.path(self.name)) + d = h.data() + assert isinstance(d, str) + assert d == self.BINARY_DATA + + def test_write_binary_file_hunk(self): + outfile = self.path('out') + h = FileHunk(self.path(self.name)) + h.save(outfile) + + with open(outfile, 'rb') as f: + d = f.read() + + assert isinstance(d, str) + assert d == self.BINARY_DATA + + def test_write_memory_hunk(self): + outfile = self.path('out') + h = MemoryHunk(self.BINARY_DATA) + h.save(outfile) + + with open(outfile, 'rb') as f: + d = f.read() + + assert isinstance(d, str) + assert d == self.BINARY_DATA