From 49daf9def1c480320cf8b63e11fa53a2dda1fe00 Mon Sep 17 00:00:00 2001 From: Adam Maris Date: Fri, 28 Jun 2019 15:32:02 +0200 Subject: [PATCH 1/2] use utf-8 codec if ensure_ascii is False to avoid UnicodeError --- cortexutils/worker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cortexutils/worker.py b/cortexutils/worker.py index 2b6e5a5..ade513c 100644 --- a/cortexutils/worker.py +++ b/cortexutils/worker.py @@ -119,8 +119,13 @@ def __write_output(self, data, ensure_ascii=False): os.makedirs('%s/output' % self.job_directory) except: pass - with open('%s/output/output.json' % self.job_directory, mode='w') as f_output: - json.dump(data, f_output, ensure_ascii=ensure_ascii) + + f_output = open('%s/output/output.json' % self.job_directory, mode='w') + + if not ensure_ascii: + f_output = codecs.getwriter('utf-8')(f_output, 'strict') + + json.dump(data, f_output, ensure_ascii=ensure_ascii) def get_data(self): """Wrapper for getting data from input dict. From f10974632c4bf62c2d51ab1bc0afea44d880802e Mon Sep 17 00:00:00 2001 From: Adam Maris Date: Mon, 25 Nov 2019 16:47:19 +0100 Subject: [PATCH 2/2] support both python2's non-ascii unicode literals and utf-encoded strings --- cortexutils/worker.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cortexutils/worker.py b/cortexutils/worker.py index ade513c..5e92250 100644 --- a/cortexutils/worker.py +++ b/cortexutils/worker.py @@ -122,10 +122,14 @@ def __write_output(self, data, ensure_ascii=False): f_output = open('%s/output/output.json' % self.job_directory, mode='w') - if not ensure_ascii: - f_output = codecs.getwriter('utf-8')(f_output, 'strict') + try: + json.dump(data, f_output, ensure_ascii=ensure_ascii) + except UnicodeEncodeError: + f_output.seek(0) + f_writer = codecs.getwriter('utf-8')(f_output, 'strict') + json.dump(data, f_writer, ensure_ascii=ensure_ascii) - json.dump(data, f_output, ensure_ascii=ensure_ascii) + f_output.close() def get_data(self): """Wrapper for getting data from input dict.