From 62711793c343d1c9af01ab8fe02ede064b38d07f Mon Sep 17 00:00:00 2001 From: Scisco Date: Mon, 16 Mar 2015 16:18:32 -0400 Subject: [PATCH 01/35] test develop and feature branches --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6dad35c..506b99a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: python branches: only: - - /^v0\.5.*$/ + - deveop + - /^feature*$/ python: - '2.7' From 6de3f21e21617dcada9326f95c500ea73ea384f6 Mon Sep 17 00:00:00 2001 From: Scisco Date: Mon, 16 Mar 2015 17:14:26 -0400 Subject: [PATCH 02/35] continuous download and process --- landsat/landsat.py | 48 +++++++++++++++++++++++++---------- landsat/tests/test_landsat.py | 6 +++++ 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index c23a24c..eff7e57 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -6,6 +6,7 @@ import argparse import textwrap import json +from os.path import join from dateutil.parser import parse import pycurl @@ -16,6 +17,7 @@ from mixins import VerbosityMixin from image import Process, FileDoesNotExist from __init__ import __version__ +import settings DESCRIPTION = """Landsat-util is a command line utility that makes it easy to @@ -127,9 +129,12 @@ def args_options(): parser_download.add_argument('-b', '--bands', help='If you specify bands, landsat-util will try to download ' 'the band from S3. If the band does not exist, an error is returned') parser_download.add_argument('-d', '--dest', help='Destination path') + parser_download.add_argument('-p', '--process', help='Process the image after download', action='store_true') + parser_download.add_argument('--pansharpen', action='store_true', + help='Whether to also pansharpen the process ' + 'image. Pan sharpening takes a long time') - parser_process = subparsers.add_parser('process', - help='Process Landsat imagery') + parser_process = subparsers.add_parser('process', help='Process Landsat imagery') parser_process.add_argument('path', help='Path to the compressed image file') parser_process.add_argument('--pansharpen', action='store_true', @@ -153,17 +158,7 @@ def main(args): if args: if args.subs == 'process': verbose = True if args.verbose else False - try: - bands = convert_to_integer_list(args.bands) - p = Process(args.path, bands=bands, verbose=verbose) - except IOError: - exit("Zip file corrupted", 1) - except FileDoesNotExist as e: - exit(e.message, 1) - - stored = p.run(args.pansharpen) - - exit("The output is stored at %s" % stored) + process_image(args.path, args.bands, verbose, args.pansharpen) elif args.subs == 'search': @@ -204,11 +199,36 @@ def main(args): d = Downloader(download_dir=args.dest) try: if d.download(args.scenes, convert_to_integer_list(args.bands)): - exit('Download Completed', 0) + if args.process: + if args.dest: + path = join(args.dest, args.scenes[0]) + else: + path = join(settings.DOWNLOAD_DIR, args.scenes[0]) + + if not args.bands: + path = path + '.tar.bz' + + process_image(path, args.bands, False, args.pansharpen) + else: + exit('Download Completed', 0) except IncorrectSceneId: exit('The SceneID provided was incorrect', 1) +def process_image(path, bands=None, verbose=False, pansharpen=False): + try: + bands = convert_to_integer_list(bands) + p = Process(path, bands=bands, verbose=verbose) + except IOError: + exit("Zip file corrupted", 1) + except FileDoesNotExist as e: + exit(e.message, 1) + + stored = p.run(pansharpen) + + exit("The output is stored at %s" % stored) + + def __main__(): global parser diff --git a/landsat/tests/test_landsat.py b/landsat/tests/test_landsat.py index e4d3215..56880b4 100644 --- a/landsat/tests/test_landsat.py +++ b/landsat/tests/test_landsat.py @@ -88,6 +88,12 @@ def test_download_incorrect(self): self.system_exit(args, 1) + def test_download_process_continuous(self): + """Test download and process commands together""" + + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.temp_folder, '-p'] + self.system_exit(args, 0) + def test_process_correct(self): """Test process command with correct input""" args = ['process', self.landsat_image] From f4c9f47df240515ee430bf4db083023cff2a75fa Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 11:38:33 -0400 Subject: [PATCH 03/35] new s3 uploader class --- landsat/uploader.py | 186 ++++++++++++++++++++++++++++++++++++++++++ requirements/base.txt | 1 + 2 files changed, 187 insertions(+) create mode 100644 landsat/uploader.py diff --git a/landsat/uploader.py b/landsat/uploader.py new file mode 100644 index 0000000..466a79c --- /dev/null +++ b/landsat/uploader.py @@ -0,0 +1,186 @@ +# Boto Uploader +# Landsat Util +# License: CC0 1.0 Universal + +# The S3 uploader is a fork of pys3upload (https://github.com/leetreveil/pys3upload) + +from __future__ import print_function, division + +import os +import sys +import time +import threading +import contextlib +import Queue +from multiprocessing import pool +try: + import cStringIO + StringIO = cStringIO +except ImportError: + import StringIO + +from boto.s3.connection import S3Connection +from mixins import VerbosityMixin + +STREAM = sys.stderr + + +class Uploader(VerbosityMixin): + + progress_template = \ + 'File Size:%(size)4d MB | Uploaded:%(uploaded)4d MB' + ' ' * 8 + + def __init__(self, key=None, secret=None, host=None): + self.key = key + self.secret = secret + self.source_size = 0 + self.conn = S3Connection(key, secret, host=host) + + def run(self, bucket_name, filename, path): + + f = open(path, 'rb') + self.source_size = os.stat(path).st_size + total_dict = {} + + def cb(part_no, uploaded, total): + + total_dict[part_no] = uploaded + + params = { + 'uploaded': round(sum(total_dict.values()) / 1048576, 0), + 'size': round(self.source_size / 1048576, 0), + } + + p = (self.progress_template + '\r') % params + + STREAM.write(p) + STREAM.flush() + + self.output('Uploading to S3', normal=True, arrow=True) + upload(bucket_name, self.key, self.secret, + data_collector(f.readlines()), filename, cb, + threads=25, replace=True, secure=True, connection=self.conn) + + print('\n') + self.output('Uploaded Completed', normal=True, arrow=True) + + +def data_collector(iterable, def_buf_size=5242880): + ''' Buffers n bytes of data + + Args: + iterable: could be a list, generator or string + def_buf_size: number of bytes to buffer, default is 5mb + + Returns: + A generator object + ''' + buf = '' + for data in iterable: + buf += data + if len(buf) >= def_buf_size: + output = buf[:def_buf_size] + buf = buf[def_buf_size:] + yield output + if len(buf) > 0: + yield buf + + +def upload_part(upload_func, progress_cb, part_no, part_data): + num_retries = 5 + + def _upload_part(retries_left=num_retries): + try: + with contextlib.closing(StringIO.StringIO(part_data)) as f: + f.seek(0) + cb = lambda c, t: progress_cb(part_no, c, t) if progress_cb else None + upload_func(f, part_no, cb=cb, num_cb=100) + except Exception, exc: + retries_left -= 1 + if retries_left > 0: + return _upload_part(retries_left=retries_left) + else: + return threading.ThreadError(repr(threading.current_thread()) + ' ' + repr(exc)) + return _upload_part() + + +def upload(bucket, aws_access_key, aws_secret_key, + iterable, key, progress_cb=None, + threads=5, replace=False, secure=True, + connection=None): + ''' Upload data to s3 using the s3 multipart upload API. + + Args: + bucket: name of s3 bucket + aws_access_key: aws access key + aws_secret_key: aws secret key + iterable: The data to upload. Each 'part' in the list + will be uploaded in parallel. Each part must be at + least 5242880 bytes (5mb). + key: the name of the key to create in the s3 bucket + progress_cb: will be called with (part_no, uploaded, total) + each time a progress update is available. + threads: the number of threads to use while uploading. (Default is 5) + replace: will replace the key in s3 if set to true. (Default is false) + secure: use ssl when talking to s3. (Default is true) + connection: used for testing + ''' + + if not connection: + from boto.s3.connection import S3Connection as connection + c = connection(aws_access_key, aws_secret_key, is_secure=secure) + else: + c = connection + + b = c.get_bucket(bucket) + + if not replace and b.lookup(key): + raise Exception('s3 key ' + key + ' already exists') + + multipart_obj = b.initiate_multipart_upload(key) + err_queue = Queue.Queue() + lock = threading.Lock() + upload.counter = 0 + + try: + tpool = pool.ThreadPool(processes=threads) + + def check_errors(): + try: + exc = err_queue.get(block=False) + except Queue.Empty: + pass + else: + raise exc + + def waiter(): + while upload.counter >= threads: + check_errors() + time.sleep(0.1) + + def cb(err): + if err: + err_queue.put(err) + with lock: + upload.counter -= 1 + + args = [multipart_obj.upload_part_from_file, progress_cb] + + for part_no, part in enumerate(iterable): + part_no += 1 + tpool.apply_async(upload_part, args + [part_no, part], callback=cb) + with lock: + upload.counter += 1 + waiter() + + tpool.close() + tpool.join() + # Check for thread errors before completing the upload, + # sometimes an error can be left unchecked until we + # get to this point. + check_errors() + multipart_obj.complete_upload() + except: + multipart_obj.cancel_upload() + tpool.terminate() + raise diff --git a/requirements/base.txt b/requirements/base.txt index 3bd6788..264f9c0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,3 +7,4 @@ six==1.9.0 scikit-image==0.10.1 homura==0.1.0 scipy==0.15.1 +boto==2.36.0 From 1e62469974e4f0d83b00499a22e92a135d98784b Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 11:42:01 -0400 Subject: [PATCH 04/35] add new flags to help text --- landsat/landsat.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index eff7e57..46f75a7 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -66,6 +66,11 @@ -d, --dest Destination path + -p, --process Process the image after download + + --pansharpen Whether to also pansharpen the processed image. + Pansharpening requires larger memory + Process: landsat.py process path [-h] [-b --bands] [-p --pansharpen] @@ -77,8 +82,8 @@ Default: Natural colors (432) Example --bands 432 - -p --pansharpen Whether to also pansharpen the process image. - Pansharpening takes a long time + --pansharpen Whether to also pansharpen the process image. + Pansharpening requires larger memory -v, --verbose Show verbose output @@ -132,14 +137,14 @@ def args_options(): parser_download.add_argument('-p', '--process', help='Process the image after download', action='store_true') parser_download.add_argument('--pansharpen', action='store_true', help='Whether to also pansharpen the process ' - 'image. Pan sharpening takes a long time') + 'image. Pansharpening requires larger memory') parser_process = subparsers.add_parser('process', help='Process Landsat imagery') parser_process.add_argument('path', help='Path to the compressed image file') parser_process.add_argument('--pansharpen', action='store_true', help='Whether to also pansharpen the process ' - 'image. Pan sharpening takes a long time') + 'image. Pansharpening requires larger memory') parser_process.add_argument('-b', '--bands', help='specify band combinations. Default is 432' 'Example: --bands 321') parser_process.add_argument('-v', '--verbose', action='store_true', From 4202774c2e75873e080333af2ed3e84b406790eb Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 12:04:09 -0400 Subject: [PATCH 05/35] add upload commands to landsat cli --- landsat/landsat.py | 59 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 46f75a7..bc69d5e 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -13,7 +13,8 @@ from downloader import Downloader, IncorrectSceneId from search import Search -from utils import reformat_date, convert_to_integer_list, timer, exit +from uploader import Uploader +from utils import reformat_date, convert_to_integer_list, timer, exit, get_file from mixins import VerbosityMixin from image import Process, FileDoesNotExist from __init__ import __version__ @@ -71,6 +72,18 @@ --pansharpen Whether to also pansharpen the processed image. Pansharpening requires larger memory + -u --upload Upload to S3 after the image processing completed + + --key Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as + Environment Variables) + + --secret Amazon S3 Secret Key (You can also be set AWS_SECRET_ACCESS_KEY as + Environment Variables) + + --bucket Bucket name (required if uploading to s3) + + --region URL to S3 region e.g. s3-us-west-2.amazonaws.com + Process: landsat.py process path [-h] [-b --bands] [-p --pansharpen] @@ -88,6 +101,18 @@ -v, --verbose Show verbose output -h, --help Show this help message and exit + + -u --upload Upload to S3 after the image processing completed + + --key Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as + Environment Variables) + + --secret Amazon S3 Secret Key (You can also be set AWS_SECRET_ACCESS_KEY as + Environment Variables) + + --bucket Bucket name (required if uploading to s3) + + --region URL to S3 region e.g. s3-us-west-2.amazonaws.com """ @@ -138,6 +163,14 @@ def args_options(): parser_download.add_argument('--pansharpen', action='store_true', help='Whether to also pansharpen the process ' 'image. Pansharpening requires larger memory') + parser_download.add_argument('-u', '--upload', action='store_true', + help='Upload to S3 after the image processing completed') + parser_download.add_argument('--key', help='Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as ' + 'Environment Variables)') + parser_download.add_argument('--secret', help='Amazon S3 Secret Key (You can also be set AWS_SECRET_ACCESS_KEY ' + 'as Environment Variables)') + parser_download.add_argument('--bucket', help='Bucket name (required if uploading to s3)') + parser_download.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com') parser_process = subparsers.add_parser('process', help='Process Landsat imagery') parser_process.add_argument('path', @@ -149,6 +182,14 @@ def args_options(): 'Example: --bands 321') parser_process.add_argument('-v', '--verbose', action='store_true', help='Turn on verbosity') + parser_process.add_argument('-u', '--upload', action='store_true', + help='Upload to S3 after the image processing completed') + parser_process.add_argument('--key', help='Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as ' + 'Environment Variables)') + parser_process.add_argument('--secret', help='Amazon S3 Secret Key (You can also be set AWS_SECRET_ACCESS_KEY ' + 'as Environment Variables)') + parser_process.add_argument('--bucket', help='Bucket name (required if uploading to s3)') + parser_process.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com') return parser @@ -163,7 +204,13 @@ def main(args): if args: if args.subs == 'process': verbose = True if args.verbose else False - process_image(args.path, args.bands, verbose, args.pansharpen) + stored = process_image(args.path, args.bands, verbose, args.pansharpen) + + if args.upload: + u = Uploader(args.key, args.secret, args.region) + u.run(args.bucket, get_file(stored), stored) + + exit("The output is stored at %s" % stored) elif args.subs == 'search': @@ -213,7 +260,9 @@ def main(args): if not args.bands: path = path + '.tar.bz' - process_image(path, args.bands, False, args.pansharpen) + stored = process_image(path, args.bands, False, args.pansharpen) + + exit("The output is stored at %s" % stored) else: exit('Download Completed', 0) except IncorrectSceneId: @@ -229,9 +278,7 @@ def process_image(path, bands=None, verbose=False, pansharpen=False): except FileDoesNotExist as e: exit(e.message, 1) - stored = p.run(pansharpen) - - exit("The output is stored at %s" % stored) + return p.run(pansharpen) def __main__(): From 84833c7b479d5f22824fb14c24c17f1ff7807de2 Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 12:43:00 -0400 Subject: [PATCH 06/35] tests for uploader --- landsat/tests/mocks.py | 28 +++++++++ landsat/tests/samples/mock_upload | 1 + landsat/tests/test_uploader.py | 97 +++++++++++++++++++++++++++++++ requirements/dev.txt | 1 + 4 files changed, 127 insertions(+) create mode 100644 landsat/tests/mocks.py create mode 100644 landsat/tests/samples/mock_upload create mode 100644 landsat/tests/test_uploader.py diff --git a/landsat/tests/mocks.py b/landsat/tests/mocks.py new file mode 100644 index 0000000..2fce9fa --- /dev/null +++ b/landsat/tests/mocks.py @@ -0,0 +1,28 @@ +state = {} + +class MockBotoS3MultipartUpload(): + def __init__(self): + self.data = state['mock_boto_s3_multipart_upload_data'] + + def upload_part_from_file(self, f, part_no, cb=None, num_cb=None): + self.data.append(f.read()) + + def complete_upload(self): + pass + + def cancel_upload(self): + pass + +class MockBotoS3Bucket(): + def lookup(self, key): + pass + + def initiate_multipart_upload(self, key): + return MockBotoS3MultipartUpload() + +class S3Connection(): + def __init__(self, key, secret, is_secure=None, host=None): + pass + + def get_bucket(self, bucket_name): + return MockBotoS3Bucket() diff --git a/landsat/tests/samples/mock_upload b/landsat/tests/samples/mock_upload new file mode 100644 index 0000000..90d2950 --- /dev/null +++ b/landsat/tests/samples/mock_upload @@ -0,0 +1 @@ +111111 diff --git a/landsat/tests/test_uploader.py b/landsat/tests/test_uploader.py new file mode 100644 index 0000000..ac7e712 --- /dev/null +++ b/landsat/tests/test_uploader.py @@ -0,0 +1,97 @@ +# Landsat Util +# License: CC0 1.0 Universal + +# Some of the tests are from pys3upload (https://github.com/leetreveil/pys3upload) + +"""Tests for uploader""" + +import os +import sys +import unittest +import threading + +import mock + +try: + from landsat.uploader import Uploader, upload, upload_part, data_collector + import landsat.tests.mocks as mocks +except ImportError: + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../landsat'))) + from landsat.uploader import Uploader + import landsat.tests.mocks as mocks + + +class TestUploader(unittest.TestCase): + + @mock.patch('landsat.uploader.S3Connection', mocks.S3Connection) + def test_upload_to_s3(self): + mocks.state['mock_boto_s3_multipart_upload_data'] = [] + base_dir = os.path.abspath(os.path.dirname(__file__)) + landsat_image = os.path.join(base_dir, 'samples/mock_upload') + f = open(landsat_image, 'rb').readlines() + + u = Uploader('some_key', 'some_secret') + u.run('some bucket', 'mock_upload', landsat_image) + + self.assertEqual(mocks.state['mock_boto_s3_multipart_upload_data'], f) + + +class upload_tests(unittest.TestCase): + + def test_should_be_able_to_upload_data(self): + input = ['12', '345'] + mocks.state['mock_boto_s3_multipart_upload_data'] = [] + conn = mocks.S3Connection('some_key', 'some_secret', True) + upload('test_bucket', 'some_key', 'some_secret', input, 'some_key', connection=conn) + self.assertEqual(mocks.state['mock_boto_s3_multipart_upload_data'], ['12', '345']) + + +class upload_part_tests(unittest.TestCase): + + def test_should_return_error_when_upload_func_raises_error(self): + def upload_func(*args, **kwargs): + raise Exception() + + with self.assertRaises(threading.ThreadError): + raise upload_part(upload_func, '_', '_', '_') + + def test_should_retry_upload_five_times(self): + counter = [0] + + def upload_func(*args, **kwargs): + counter[0] += 1 + raise Exception() + + upload_part(upload_func, '_', '_', '_') + self.assertEqual(counter[0], 5) + + +class doc_collector_tests(unittest.TestCase): + + def test_should_be_able_to_read_every_byte_of_data(self): + input = ['12345'] + result = list(data_collector(input, def_buf_size=3)) + self.assertEqual(result, ['123', '45']) + + def test_should_be_able_to_read_single_yield(self): + input = ['123'] + result = list(data_collector(input, def_buf_size=3)) + self.assertEqual(result, ['123']) + + def test_should_be_able_to_yield_data_less_than_buffer_size(self): + input = ['123'] + result = list(data_collector(input, def_buf_size=6)) + self.assertEqual(result, ['123']) + + def test_a_single_item_should_still_be_buffered_even_if_it_is_above_the_buffer_size(self): + input = ['123456'] + result = list(data_collector(input, def_buf_size=3)) + self.assertEqual(result, ['123', '456']) + + def test_should_return_rest_of_data_on_last_iteration(self): + input = ['1234', '56'] + result = list(data_collector(input, def_buf_size=3)) + self.assertEqual(result, ['123', '456']) + +if __name__ == '__main__': + unittest.main() diff --git a/requirements/dev.txt b/requirements/dev.txt index 74229be..d0c7e23 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,3 +4,4 @@ nose==1.3.3 coverage==3.7.1 Sphinx==1.2.3 wheel==0.23.0 +mock==1.0.1 From a3dc40f5fa712ce5aa9e2f9d4d65bfc47952b0f1 Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 13:08:22 -0400 Subject: [PATCH 07/35] upload sequence was missing from download sub command --- landsat/landsat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/landsat/landsat.py b/landsat/landsat.py index bc69d5e..87d53b6 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -262,6 +262,10 @@ def main(args): stored = process_image(path, args.bands, False, args.pansharpen) + if args.upload: + u = Uploader(args.key, args.secret, args.region) + u.run(args.bucket, get_file(stored), stored) + exit("The output is stored at %s" % stored) else: exit('Download Completed', 0) From 90429d876d64a0ae63e9d71cce25ac5b4184650e Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 13:26:29 -0400 Subject: [PATCH 08/35] fix misspelling --- landsat/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/uploader.py b/landsat/uploader.py index 466a79c..9f2745b 100644 --- a/landsat/uploader.py +++ b/landsat/uploader.py @@ -62,7 +62,7 @@ def cb(part_no, uploaded, total): threads=25, replace=True, secure=True, connection=self.conn) print('\n') - self.output('Uploaded Completed', normal=True, arrow=True) + self.output('Upload Completed', normal=True, arrow=True) def data_collector(iterable, def_buf_size=5242880): From 56eaa79b06be59f428eccc5703bd21abb6aa637c Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 17 Mar 2015 13:29:06 -0400 Subject: [PATCH 09/35] reduce threads to 10 --- landsat/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/uploader.py b/landsat/uploader.py index 9f2745b..bbbc151 100644 --- a/landsat/uploader.py +++ b/landsat/uploader.py @@ -59,7 +59,7 @@ def cb(part_no, uploaded, total): self.output('Uploading to S3', normal=True, arrow=True) upload(bucket_name, self.key, self.secret, data_collector(f.readlines()), filename, cb, - threads=25, replace=True, secure=True, connection=self.conn) + threads=10, replace=True, secure=True, connection=self.conn) print('\n') self.output('Upload Completed', normal=True, arrow=True) From d532d5eab450863e4d2ea4c11081a960aaa43ef2 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:01:40 -0400 Subject: [PATCH 10/35] better spacing --- landsat/tests/mocks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/landsat/tests/mocks.py b/landsat/tests/mocks.py index 2fce9fa..521785c 100644 --- a/landsat/tests/mocks.py +++ b/landsat/tests/mocks.py @@ -1,5 +1,6 @@ state = {} + class MockBotoS3MultipartUpload(): def __init__(self): self.data = state['mock_boto_s3_multipart_upload_data'] @@ -13,6 +14,7 @@ def complete_upload(self): def cancel_upload(self): pass + class MockBotoS3Bucket(): def lookup(self, key): pass @@ -20,6 +22,7 @@ def lookup(self, key): def initiate_multipart_upload(self, key): return MockBotoS3MultipartUpload() + class S3Connection(): def __init__(self, key, secret, is_secure=None, host=None): pass From 327c69a7dde01622bdc346171841b6049e50cdc2 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:02:26 -0400 Subject: [PATCH 11/35] updated tests with mock --- landsat/tests/test_landsat.py | 127 +++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/landsat/tests/test_landsat.py b/landsat/tests/test_landsat.py index 56880b4..6b546b2 100644 --- a/landsat/tests/test_landsat.py +++ b/landsat/tests/test_landsat.py @@ -4,12 +4,10 @@ """Tests for landsat""" import sys -import errno -import shutil import unittest import subprocess -from tempfile import mkdtemp from os.path import join, abspath, dirname +import mock try: import landsat.landsat as landsat @@ -22,95 +20,140 @@ class TestLandsat(unittest.TestCase): @classmethod def setUpClass(cls): - cls.temp_folder = mkdtemp() - cls.base_dir = abspath(dirname(__file__)) - cls.landsat_image = join(cls.base_dir, 'samples', 'test.tar.bz2') cls.parser = landsat.args_options() - @classmethod - def tearDownClass(cls): - try: - shutil.rmtree(cls.temp_folder) - shutil.rmtree(join(cls.base_dir, 'samples', 'test')) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise - - def system_exit(self, args, code): - try: - landsat.main(self.parser.parse_args(args)) - except SystemExit as e: - self.assertEqual(e.code, code) - def test_incorrect_date(self): """ Test search with incorrect date input """ args = ['search', '--start', 'berlin', '--end', 'january 10 2014'] - self.system_exit(args, 1) + self.assertEquals(landsat.main(self.parser.parse_args(args)), + ['You date format is incorrect. Please try again!', 1]) def test_too_many_results(self): """ Test when search return too many results """ args = ['search', '--cloud', '100', '-p', '205,022,206,022,204,022'] - self.system_exit(args, 1) + self.assertEquals(landsat.main(self.parser.parse_args(args)), + ['Over 100 results. Please narrow your search', 1]) def test_search_pr_correct(self): """Test Path Row search with correct input""" args = ['search', '--start', 'january 1 2013', '--end', 'january 10 2014', '-p', '008,008'] - self.system_exit(args, 0) + self.assertEquals(landsat.main(self.parser.parse_args(args)), + ['Search completed!']) def test_search_lat_lon(self): """Test Latitude Longitude search with correct input""" args = ['search', '--start', 'may 01 2013', '--end', 'may 08 2013', '--lat', '38.9107203', '--lon', '-77.0290116'] - self.system_exit(args, 0) + self.assertEquals(landsat.main(self.parser.parse_args(args)), + ['Search completed!']) def test_search_pr_wrong_input(self): """Test Path Row search with incorrect input""" args = ['search', '-p', 'what?'] - self.system_exit(args, 1) + self.assertEquals(landsat.main(self.parser.parse_args(args)), + ['Check your request and try again', 1]) - def test_download_correct(self): + @mock.patch('landsat.landsat.Downloader') + def test_download_correct(self, mock_downloader): """Test download command with correct input""" - args = ['download', 'LC80010092015051LGN00', '-b', '11,', '-d', self.temp_folder] + mock_downloader.download.return_value = True - self.system_exit(args, 0) + args = ['download', 'LC80010092015051LGN00', '-b', '11,', '-d', 'path/to/folder'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(download_dir='path/to/folder') + mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], ['11', '']) + self.assertEquals(output, ['Download Completed', 0]) def test_download_incorrect(self): """Test download command with incorrect input""" args = ['download', 'LT813600'] - self.system_exit(args, 1) + self.assertEquals(landsat.main(self.parser.parse_args(args)), + ['The SceneID provided was incorrect', 1]) - def test_download_process_continuous(self): + @mock.patch('landsat.landsat.process_image') + @mock.patch('landsat.landsat.Downloader.download') + def test_download_process_continuous(self, mock_downloader, mock_process): """Test download and process commands together""" - - args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.temp_folder, '-p'] - self.system_exit(args, 0) - - def test_process_correct(self): + mock_downloader.return_value = True + mock_process.return_value = 'image.TIF' + + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', 'path/to/folder', '-p'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + self.assertEquals(output, ["The output is stored at image.TIF"]) + + @mock.patch('landsat.landsat.Uploader') + @mock.patch('landsat.landsat.process_image') + @mock.patch('landsat.landsat.Downloader.download') + def test_download_process_continuous_with_upload(self, mock_downloader, mock_process, mock_upload): + """Test download and process commands together""" + mock_downloader.return_value = True + mock_process.return_value = 'image.TIF' + mock_upload.run.return_value = True + + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', 'path/to/folder', '-p', + '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + mock_upload.assert_called_with('somekey', 'somesecret', 'this') + mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF') + self.assertEquals(output, ["The output is stored at image.TIF"]) + + @mock.patch('landsat.landsat.process_image') + @mock.patch('landsat.landsat.Downloader.download') + def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock_process): + """Test download and process commands together""" + mock_downloader.return_value = True + mock_process.return_value = 'image.TIF' + + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', 'path/to/folder', '-p', + '-u', '--region', 'whatever'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) + self.assertEquals(output, ['Could not authenticate with AWS', 1]) + + @mock.patch('landsat.landsat.process_image') + def test_process_correct(self, mock_process): """Test process command with correct input""" - args = ['process', self.landsat_image] + mock_process.return_value = 'image.TIF' + + args = ['process', 'path/to/folder/LC80010092015051LGN00'] + output = landsat.main(self.parser.parse_args(args)) - self.system_exit(args, 0) + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False) + self.assertEquals(output, ["The output is stored at image.TIF"]) - def test_process_correct_pansharpen(self): + @mock.patch('landsat.landsat.process_image') + def test_process_correct_pansharpen(self, mock_process): """Test process command with correct input and pansharpening""" - args = ['process', '--pansharpen', self.landsat_image] + mock_process.return_value = 'image.TIF' - self.system_exit(args, 0) + args = ['process', '--pansharpen', 'path/to/folder/LC80010092015051LGN00'] + output = landsat.main(self.parser.parse_args(args)) + + mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True) + self.assertEquals(output, ["The output is stored at image.TIF"]) def test_process_incorrect(self): """Test process command with incorrect input""" args = ['process', 'whatever'] - self.system_exit(args, 1) + try: + landsat.main(self.parser.parse_args(args)) + except SystemExit as e: + self.assertEqual(e.code, 1) def check_command_line(self): """ Check if the commandline performs correctly """ From 70de06c90dd494c936d7541526e9b4237b74e40e Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:03:24 -0400 Subject: [PATCH 12/35] convert exits to returns resolves #53 --- landsat/landsat.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 87d53b6..2e2698b 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -7,9 +7,11 @@ import textwrap import json from os.path import join +from urllib2 import URLError from dateutil.parser import parse import pycurl +from boto.exception import NoAuthHandlerFound from downloader import Downloader, IncorrectSceneId from search import Search @@ -210,7 +212,7 @@ def main(args): u = Uploader(args.key, args.secret, args.region) u.run(args.bucket, get_file(stored), stored) - exit("The output is stored at %s" % stored) + return ["The output is stored at %s" % stored] elif args.subs == 'search': @@ -220,7 +222,7 @@ def main(args): if args.end: args.end = reformat_date(parse(args.end)) except (TypeError, ValueError): - exit("You date format is incorrect. Please try again!", 1) + return ["You date format is incorrect. Please try again!", 1] s = Search() @@ -228,7 +230,7 @@ def main(args): lat = float(args.lat) if args.lat else None lon = float(args.lon) if args.lon else None except ValueError: - exit("The latitude and longitude values must be valid numbers", 1) + return ["The latitude and longitude values must be valid numbers", 1] result = s.search(paths_rows=args.pathrow, lat=lat, @@ -241,12 +243,12 @@ def main(args): if result['status'] == 'SUCCESS': v.output('%s items were found' % result['total'], normal=True, arrow=True) if result['total'] > 100: - exit('Over 100 results. Please narrow your search', 1) + return ['Over 100 results. Please narrow your search', 1] else: v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green') - exit('Search completed!') + return ['Search completed!'] elif result['status'] == 'error': - exit(result['message'], 1) + return [result['message'], 1] elif args.subs == 'download': d = Downloader(download_dir=args.dest) try: @@ -263,14 +265,19 @@ def main(args): stored = process_image(path, args.bands, False, args.pansharpen) if args.upload: - u = Uploader(args.key, args.secret, args.region) + try: + u = Uploader(args.key, args.secret, args.region) + except NoAuthHandlerFound: + return ["Could not authenticate with AWS", 1] + except URLError: + return ["Connection timeout. Probably the region parameter is incorrect", 1] u.run(args.bucket, get_file(stored), stored) - exit("The output is stored at %s" % stored) + return ["The output is stored at %s" % stored] else: - exit('Download Completed', 0) + return ['Download Completed', 0] except IncorrectSceneId: - exit('The SceneID provided was incorrect', 1) + return ['The SceneID provided was incorrect', 1] def process_image(path, bands=None, verbose=False, pansharpen=False): @@ -282,7 +289,7 @@ def process_image(path, bands=None, verbose=False, pansharpen=False): except FileDoesNotExist as e: exit(e.message, 1) - return p.run(pansharpen) + return [p.run(pansharpen), 0] def __main__(): @@ -291,7 +298,7 @@ def __main__(): parser = args_options() args = parser.parse_args() with timer(): - main(args) + exit(*main(args)) if __name__ == "__main__": try: From b16a7206c1f1b63281300d842eb8f50f82b65efb Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:06:11 -0400 Subject: [PATCH 13/35] remove mock path at the end of the test --- landsat/tests/test_landsat.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/landsat/tests/test_landsat.py b/landsat/tests/test_landsat.py index 6b546b2..d6a74c1 100644 --- a/landsat/tests/test_landsat.py +++ b/landsat/tests/test_landsat.py @@ -6,6 +6,8 @@ import sys import unittest import subprocess +import errno +import shutil from os.path import join, abspath, dirname import mock @@ -21,6 +23,15 @@ class TestLandsat(unittest.TestCase): @classmethod def setUpClass(cls): cls.parser = landsat.args_options() + cls.mock_path = 'path/to/folder' + + @classmethod + def tearDownClass(cls): + try: + shutil.rmtree('path') + except OSError as exc: + if exc.errno != errno.ENOENT: + raise def test_incorrect_date(self): """ Test search with incorrect date input """ @@ -66,9 +77,9 @@ def test_download_correct(self, mock_downloader): """Test download command with correct input""" mock_downloader.download.return_value = True - args = ['download', 'LC80010092015051LGN00', '-b', '11,', '-d', 'path/to/folder'] + args = ['download', 'LC80010092015051LGN00', '-b', '11,', '-d', self.mock_path] output = landsat.main(self.parser.parse_args(args)) - mock_downloader.assert_called_with(download_dir='path/to/folder') + mock_downloader.assert_called_with(download_dir=self.mock_path) mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], ['11', '']) self.assertEquals(output, ['Download Completed', 0]) @@ -86,7 +97,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): mock_downloader.return_value = True mock_process.return_value = 'image.TIF' - args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', 'path/to/folder', '-p'] + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False) @@ -101,7 +112,7 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro mock_process.return_value = 'image.TIF' mock_upload.run.return_value = True - args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', 'path/to/folder', '-p', + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) @@ -117,7 +128,7 @@ def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock mock_downloader.return_value = True mock_process.return_value = 'image.TIF' - args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', 'path/to/folder', '-p', + args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p', '-u', '--region', 'whatever'] output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2']) From 1e0364d3da9e5b3b19ea3e3917803237392b6bc6 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:35:28 -0400 Subject: [PATCH 14/35] add mock to download tests --- landsat/tests/test_download.py | 83 ++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/landsat/tests/test_download.py b/landsat/tests/test_download.py index 51ae8cc..16143cb 100644 --- a/landsat/tests/test_download.py +++ b/landsat/tests/test_download.py @@ -10,12 +10,14 @@ import unittest from tempfile import mkdtemp +import mock + try: - from landsat.downloader import Downloader + from landsat.downloader import Downloader, RemoteFileDoesntExist, IncorrectSceneId from landsat.settings import GOOGLE_STORAGE, S3_LANDSAT except ImportError: sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../landsat'))) - from landsat.downloader import Downloader + from landsat.downloader import Downloader, RemoteFileDoesntExist, IncorrectSceneId from landsat.settings import GOOGLE_STORAGE, S3_LANDSAT @@ -43,57 +45,64 @@ def assertSize(self, url, path): self.assertEqual(remote_size, download_size) - def test_download(self): - sat = self.d.scene_interpreter(self.scene) - url = self.d.google_storage_url(sat) - self.d.download_dir = self.temp_folder + @mock.patch('landsat.downloader.fetch') + def test_download(self, mock_fetch): + mock_fetch.return_value = True # download one list self.d.download([self.scene]) - self.assertSize(url, os.path.join(self.temp_folder, self.scene + '.tar.bz')) + self.assertTrue(self.d.download([self.scene])) - # pass string instead of list + # # pass string instead of list self.assertRaises(Exception, self.d.download, self.scene) - # pass multiple sceneIDs + # # pass multiple sceneIDs self.d.download([self.scene, self.scene]) - self.assertSize(url, os.path.join(self.temp_folder, self.scene + '.tar.bz')) + self.assertTrue(self.d.download([self.scene])) - # pass band along with sceneID + # # pass band along with sceneID self.d.download([self.scene_s3], bands=[11]) - filename = '%s_B11.TIF' % self.scene_s3 - sat = self.d.scene_interpreter(self.scene_s3) - url = self.d.amazon_s3_url(sat, filename) - self.assertSize(url, os.path.join(self.temp_folder, self.scene_s3, filename)) + self.assertTrue(self.d.download([self.scene])) - # pass band as string + # # pass band as string self.assertRaises(Exception, self.d.download, self.scene, 4) - def test_google_storage(self): - sat = self.d.scene_interpreter(self.scene) - url = self.d.google_storage_url(sat) - self.d.google_storage(self.scene, self.temp_folder) + @mock.patch('landsat.downloader.fetch') + def test_google_storage(self, mock_fetch): + mock_fetch.return_value = True + + # If the file exist + self.assertTrue(self.d.google_storage(self.scene, self.temp_folder)) - self.assertSize(url, os.path.join(self.temp_folder, self.scene + '.tar.bz')) + # If scene id is incorrect + self.assertRaises(IncorrectSceneId, self.d.google_storage, 'somerandomscene', self.temp_folder) + + # If scene id doesn't exist + self.assertRaises(RemoteFileDoesntExist, self.d.google_storage, 'LG21360082013227LGN01', + self.temp_folder) + + @mock.patch('landsat.downloader.fetch') + def test_amazon_s3(self, mock_fetch): + mock_fetch.return_value = True - def test_amazon_s3(self): scene = self.scene_s3 - sat = self.d.scene_interpreter(scene) - filename = '%s_B11.TIF' % scene - url = self.d.amazon_s3_url(sat, filename) + self.assertTrue(self.d.amazon_s3(scene, 11, self.temp_folder)) + + # If scene id is incorrect + self.assertRaises(IncorrectSceneId, self.d.amazon_s3, 'somerandomscene', 11, self.temp_folder) - self.d.amazon_s3(scene, 11, self.temp_folder) + # If scene id doesn't exist + self.assertRaises(RemoteFileDoesntExist, self.d.amazon_s3, 'LT81360082013127LGN01', 33, self.temp_folder) - self.assertSize(url, os.path.join(self.temp_folder, filename)) + @mock.patch('landsat.downloader.fetch') + def test_fetch(self, mock_fetch): + mock_fetch.return_value = True - def test_fetch(self): sat = self.d.scene_interpreter(self.scene) url = self.d.google_storage_url(sat) - # download - self.d.fetch(url, self.temp_folder, self.scene) - self.assertSize(url, os.path.join(self.temp_folder, self.scene + '.tar.bz')) + self.assertTrue(self.d.fetch(url, self.temp_folder, self.scene)) def test_remote_file_size(self): @@ -119,14 +128,22 @@ def test_amazon_s3_url(self): self.assertEqual(expect, string) def test_remote_file_exist(self): - # Test a url that doesn't exist + # Test a S3 url that exists self.assertTrue(self.d.remote_file_exists(os.path.join(S3_LANDSAT, 'L8/003/017/LC80030172015001L' 'GN00/LC80030172015001LGN00_B6.TIF'))) - # Test a url that exist + # Test a S3 url that doesn't exist self.assertFalse(self.d.remote_file_exists(os.path.join(S3_LANDSAT, 'L8/003/017/LC80030172015001L' 'GN00/LC80030172015001LGN00_B34.TIF'))) + # Test a Google Storage url that doesn't exist + self.assertFalse(self.d.remote_file_exists(os.path.join(GOOGLE_STORAGE, 'L8/003/017/LC80030172015001L' + 'GN00/LC80030172015001LGN00_B6.TIF'))) + + # Test a Google Storage url that exists + self.assertTrue(self.d.remote_file_exists(os.path.join(GOOGLE_STORAGE, + 'L8/003/017/LC80030172015001LGN00.tar.bz'))) + def test_scene_interpreter(self): # Test with correct input scene = 'LC80030172015001LGN00' From fc8ec421cf03488d74ccd1ae1ba81fed1246ab7f Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:35:42 -0400 Subject: [PATCH 15/35] add missing raise statement --- landsat/downloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index 7a8f6c9..30943b8 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -63,10 +63,10 @@ def google_storage(self, scene, path): url = self.google_storage_url(sat) if self.remote_file_exists(url): - self.fetch(url, path, filename) + return self.fetch(url, path, filename) else: - RemoteFileDoesntExist('%s is not available on Google Storage' % filename) + raise RemoteFileDoesntExist('%s is not available on Google Storage' % filename) def amazon_s3(self, scene, band, path): """ Amazon S3 downloader """ @@ -76,10 +76,10 @@ def amazon_s3(self, scene, band, path): url = self.amazon_s3_url(sat, filename) if self.remote_file_exists(url): - self.fetch(url, path, filename) + return self.fetch(url, path, filename) else: - RemoteFileDoesntExist('%s is not available on Amazon S3' % filename) + raise RemoteFileDoesntExist('%s is not available on Amazon S3' % filename) def fetch(self, url, path, filename): From 42825ec7b4339babc9e7dfdda988008ea518f3be Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 12:43:56 -0400 Subject: [PATCH 16/35] tar unzipping issues resolves #55 --- landsat/image.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/landsat/image.py b/landsat/image.py index 080d9c5..50e664d 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -6,11 +6,12 @@ import sys from os.path import join import tarfile +import glob +import subprocess + import numpy import rasterio -import glob from rasterio.warp import reproject, RESAMPLING, transform - from skimage import img_as_ubyte, exposure from skimage import transform as sktransform @@ -158,7 +159,7 @@ def run(self, pansharpen=True): if i == 2: band = self._gamma_correction(band, 0.9) - + output.write_band(i+1, img_as_ubyte(band)) new_bands[i] = None @@ -254,9 +255,14 @@ def _percent_cut(self, color): def _unzip(self, src, dst, scene): """ Unzip tar files """ self.output("Unzipping %s - It might take some time" % scene, normal=True, arrow=True) - tar = tarfile.open(src) - tar.extractall(path=dst) - tar.close() + + try: + tar = tarfile.open(src, 'r') + tar.extractall(path=dst) + tar.close() + except tarfile.ReadError: + check_create_folder(dst) + subprocess.check_call(['tar', '-xf', src, '-C', dst]) def _get_full_filename(self, band): From 8a412ceb774e60b3257a9b3e7a90442a1456dfe1 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 13:46:10 -0400 Subject: [PATCH 17/35] few pep8 changes --- landsat/image.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/landsat/image.py b/landsat/image.py index 50e664d..45c3b6c 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -12,8 +12,10 @@ import numpy import rasterio from rasterio.warp import reproject, RESAMPLING, transform -from skimage import img_as_ubyte, exposure + from skimage import transform as sktransform +from skimage.util import img_as_ubyte +from skimage.exposure import rescale_intensity, adjust_gamma import settings from mixins import VerbosityMixin @@ -104,10 +106,17 @@ def run(self, pansharpen=True): crn = self._get_boundaries(src_data) dst_shape = src_data['shape'] - y_pixel = (max(crn['ul']['y'][1][0],crn['ur']['y'][1][0])- min(crn['lr']['y'][1][0],crn['ll']['y'][1][0]))/dst_shape[0] - x_pixel = (max(crn['lr']['x'][1][0],crn['ur']['x'][1][0]) - min(crn['ul']['x'][1][0],crn['ll']['x'][1][0]))/dst_shape[1] - - dst_transform = (min(crn['ul']['x'][1][0],crn['ll']['x'][1][0]), x_pixel, 0.0, max(crn['ul']['y'][1][0],crn['ur']['y'][1][0]), 0.0, -y_pixel) + y_pixel = (max(crn['ul']['y'][1][0], crn['ur']['y'][1][0]) - + min(crn['lr']['y'][1][0], crn['ll']['y'][1][0])) / dst_shape[0] + x_pixel = (max(crn['lr']['x'][1][0], crn['ur']['x'][1][0]) - + min(crn['ul']['x'][1][0], crn['ll']['x'][1][0])) / dst_shape[1] + + dst_transform = (min(crn['ul']['x'][1][0], crn['ll']['x'][1][0]), + x_pixel, + 0.0, + max(crn['ul']['y'][1][0], crn['ur']['y'][1][0]), + 0.0, + -y_pixel) # Delete crn since no longer needed del crn @@ -159,7 +168,6 @@ def run(self, pansharpen=True): if i == 2: band = self._gamma_correction(band, 0.9) - output.write_band(i+1, img_as_ubyte(band)) new_bands[i] = None @@ -188,14 +196,14 @@ def _pansharpenning(self, bands): return bands def _gamma_correction(self, band, value): - return exposure.adjust_gamma(band, value) + return adjust_gamma(band, value) def _color_correction(self, band, band_id): band = band.astype(numpy.uint16) self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1) p2, p98 = self._percent_cut(band) - band = exposure.rescale_intensity(band, in_range=(p2, p98)) + band = rescale_intensity(band, in_range=(p2, p98)) return band From 276692b902edd9177824b892fc78c0ad98f4208b Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 13:51:09 -0400 Subject: [PATCH 18/35] remove travis branch restriction --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 506b99a..633aa0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,5 @@ language: python -branches: - only: - - deveop - - /^feature*$/ - python: - '2.7' From 13d17a3a320c2256b395f7a5667e2ba69c61a34e Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 14:07:39 -0400 Subject: [PATCH 19/35] bump up dev version to 0.5.5 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index 14b1592..e0ca4de 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -4,4 +4,4 @@ if not settings.DEBUG: sys.tracebacklimit = 0 -__version__ = '0.5.0' +__version__ = '0.5.5' From 4d2f5e6d1a5859ffbfd5a6864da2b081391fb992 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 14:48:50 -0400 Subject: [PATCH 20/35] return file path as string not list --- landsat/landsat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 2e2698b..9f6480c 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -289,7 +289,7 @@ def process_image(path, bands=None, verbose=False, pansharpen=False): except FileDoesNotExist as e: exit(e.message, 1) - return [p.run(pansharpen), 0] + return p.run(pansharpen) def __main__(): From d47a61bc08debe93ed73f84ea1115028b1c09e7b Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 14:59:31 -0400 Subject: [PATCH 21/35] add mock for travis --- requirements/travis.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/travis.txt b/requirements/travis.txt index d8b9bbe..9f6f4d1 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -2,3 +2,5 @@ rasterio==0.18 homura==0.1.0 termcolor==1.1.0 python-coveralls +mock==1.0.1 + From a4fc9ce7537a586aecc60ac0751c161edec882ac Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 15:24:01 -0400 Subject: [PATCH 22/35] add boto to travis requirements --- requirements/travis.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/travis.txt b/requirements/travis.txt index 9f6f4d1..3d673b7 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -3,4 +3,5 @@ homura==0.1.0 termcolor==1.1.0 python-coveralls mock==1.0.1 +boto==2.36.0 From 8fa7c5429c3ec8f0c6a01d531996c4a0984488b2 Mon Sep 17 00:00:00 2001 From: Marc Farra Date: Wed, 18 Mar 2015 15:49:54 -0400 Subject: [PATCH 23/35] Switch to adaptive histogram equalization --- landsat/image.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/landsat/image.py b/landsat/image.py index 45c3b6c..db5288b 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -15,7 +15,7 @@ from skimage import transform as sktransform from skimage.util import img_as_ubyte -from skimage.exposure import rescale_intensity, adjust_gamma +from skimage.exposure import equalize_adapthist, adjust_gamma import settings from mixins import VerbosityMixin @@ -202,8 +202,7 @@ def _color_correction(self, band, band_id): band = band.astype(numpy.uint16) self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1) - p2, p98 = self._percent_cut(band) - band = rescale_intensity(band, in_range=(p2, p98)) + band = equalize_adapthist(band, clip_limit=0.02) return band From db7115dd155b5e234993f9ba09c854994ccd4623 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 17:39:37 -0400 Subject: [PATCH 24/35] don't calculate coverage --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 633aa0e..3499f55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,3 @@ install: script: - nosetests - -after_success: - - coveralls From e4853040058f033bacb9dffa345b3f14210f18c5 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 18 Mar 2015 17:40:21 -0400 Subject: [PATCH 25/35] if image is not found on amazon download from Google --- landsat/downloader.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index 30943b8..658ca5b 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -43,12 +43,14 @@ def download(self, scenes, bands=None): if isinstance(bands, list): # Create a folder to download the specific bands into path = check_create_folder(join(self.download_dir, scene)) - for band in bands: - self.amazon_s3(scene, band, path) + try: + for band in bands: + self.amazon_s3(scene, band, path) + except RemoteFileDoesntExist: + self.google_storage(scene, self.download_dir) else: raise Exception('Expected bands list') - else: - self.google_storage(scene, self.download_dir) + self.google_storage(scene, self.download_dir) return True From b557ceae29911223b9ae0558d6fe8a557fa30097 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 22 Apr 2015 15:13:31 -0400 Subject: [PATCH 26/35] updated color correction algorithm (done by @kamicut and @drewbo) --- landsat/image.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/landsat/image.py b/landsat/image.py index db5288b..0a8020f 100644 --- a/landsat/image.py +++ b/landsat/image.py @@ -15,7 +15,7 @@ from skimage import transform as sktransform from skimage.util import img_as_ubyte -from skimage.exposure import equalize_adapthist, adjust_gamma +from skimage.exposure import rescale_intensity, adjust_gamma import settings from mixins import VerbosityMixin @@ -69,6 +69,18 @@ def run(self, pansharpen=True): self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True) + # Read cloud coverage from mtl file + cloud_cover = 0 + try: + with open(self.scene_path + '/' + self.scene + '_MTL.txt', 'rU') as mtl: + lines = mtl.readlines() + for line in lines: + if 'CLOUD_COVER' in line: + cloud_cover = float(line.replace('CLOUD_COVER = ', '')) + break + except IOError: + pass + with warnings.catch_warnings(): warnings.simplefilter("ignore") with rasterio.drivers(): @@ -159,7 +171,7 @@ def run(self, pansharpen=True): for i, band in enumerate(new_bands): # Color Correction - band = self._color_correction(band, self.bands[i]) + band = self._color_correction(band, self.bands[i], 0.1, cloud_cover) # Gamma Correction if i == 0: @@ -198,13 +210,17 @@ def _pansharpenning(self, bands): def _gamma_correction(self, band, value): return adjust_gamma(band, value) - def _color_correction(self, band, band_id): + def _color_correction(self, band, band_id, low, cloud_cover): band = band.astype(numpy.uint16) self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1) - band = equalize_adapthist(band, clip_limit=0.02) - - return band + p_low, cloud_cut_low = self._percent_cut(band, low, 100 - (cloud_cover * 3 / 4)) + temp = numpy.zeros(numpy.shape(band), dtype=numpy.uint16) + cloud_divide = 65000 - cloud_cover * 100 + mask = numpy.logical_and(band < cloud_cut_low, band > 0) + temp[mask] = rescale_intensity(band[mask], in_range=(p_low, cloud_cut_low), out_range=(256, cloud_divide)) + temp[band >= cloud_cut_low] = rescale_intensity(band[band >= cloud_cut_low], out_range=(cloud_divide, 65535)) + return temp def _read_band(self, band_path): """ Reads a band with rasterio """ @@ -256,8 +272,8 @@ def _get_boundaries(self, src): return output - def _percent_cut(self, color): - return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (2, 98)) + def _percent_cut(self, color, low, high): + return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high)) def _unzip(self, src, dst, scene): """ Unzip tar files """ From f40fb7a27934bbd2e6fe758b33d9dcc567858c8e Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 22 Apr 2015 15:15:54 -0400 Subject: [PATCH 27/35] make sure to download MTL file from amazon (done by @drewbo) --- landsat/downloader.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index 658ca5b..cd9d933 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -44,7 +44,10 @@ def download(self, scenes, bands=None): # Create a folder to download the specific bands into path = check_create_folder(join(self.download_dir, scene)) try: - for band in bands: + # Always grab MTL.txt if bands are specified + bands_plus = bands + bands_plus.append('MTL') + for band in bands_plus: self.amazon_s3(scene, band, path) except RemoteFileDoesntExist: self.google_storage(scene, self.download_dir) @@ -74,7 +77,10 @@ def amazon_s3(self, scene, band, path): """ Amazon S3 downloader """ sat = self.scene_interpreter(scene) - filename = '%s_B%s.TIF' % (scene, band) + if band != 'MTL': + filename = '%s_B%s.TIF' % (scene, band) + else: + filename = '%s_%s.txt' % (scene, band) url = self.amazon_s3_url(sat, filename) if self.remote_file_exists(url): From 4dc0a403baad5dfe753a892ab34e489888c3ae6e Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 22 Apr 2015 15:54:30 -0400 Subject: [PATCH 28/35] only try amazon s3 if image from 2015 an onward --- landsat/downloader.py | 6 ++++-- landsat/landsat.py | 3 ++- landsat/tests/test_download.py | 24 ++++++++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index cd9d933..101d73a 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -39,7 +39,8 @@ def download(self, scenes, bands=None): if isinstance(scenes, list): for scene in scenes: - if bands: + # If bands are provided the image is from 2015 or later use Amazon + if (bands and int(scene[12]) > 4): if isinstance(bands, list): # Create a folder to download the specific bands into path = check_create_folder(join(self.download_dir, scene)) @@ -53,7 +54,8 @@ def download(self, scenes, bands=None): self.google_storage(scene, self.download_dir) else: raise Exception('Expected bands list') - self.google_storage(scene, self.download_dir) + else: + self.google_storage(scene, self.download_dir) return True diff --git a/landsat/landsat.py b/landsat/landsat.py index 9f6480c..c81f4d2 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -259,7 +259,8 @@ def main(args): else: path = join(settings.DOWNLOAD_DIR, args.scenes[0]) - if not args.bands: + # Keep using Google if the image is before 2015 + if (int(args.scenes[0][12]) < 5 or not args.bands): path = path + '.tar.bz' stored = process_image(path, args.bands, False, args.pansharpen) diff --git a/landsat/tests/test_download.py b/landsat/tests/test_download.py index 16143cb..6166497 100644 --- a/landsat/tests/test_download.py +++ b/landsat/tests/test_download.py @@ -53,21 +53,37 @@ def test_download(self, mock_fetch): self.d.download([self.scene]) self.assertTrue(self.d.download([self.scene])) - # # pass string instead of list + # Test if error is raised when passing scene as string instead of list self.assertRaises(Exception, self.d.download, self.scene) - # # pass multiple sceneIDs + # Test if download works when passing scenes as list self.d.download([self.scene, self.scene]) self.assertTrue(self.d.download([self.scene])) - # # pass band along with sceneID + # Test when passing band list along with sceneID self.d.download([self.scene_s3], bands=[11]) self.assertTrue(self.d.download([self.scene])) - # # pass band as string + # Test whether passing band as string raises an exception self.assertRaises(Exception, self.d.download, self.scene, 4) + @mock.patch('landsat.downloader.Downloader.amazon_s3') + @mock.patch('landsat.downloader.Downloader.google_storage') + def test_download_google_amazon(self, fake_google, fake_amazon): + """ Test whether google or amazon are correctly selected based on input """ + + fake_amazon.return_value = True + fake_google.return_value = False + + # Test if google is used when an image from 2014 is passed even if bands are provided + self.d.download([self.scene], bands=[432]) + fake_google.assert_called_with(self.scene, self.d.download_dir) + + # Test if amazon is used when an image from 2015 is passed with bands + self.d.download([self.scene_s3], bands=[432]) + fake_amazon.assert_called_with(self.scene_s3, 'MTL', self.d.download_dir + '/' + self.scene_s3) + @mock.patch('landsat.downloader.fetch') def test_google_storage(self, mock_fetch): mock_fetch.return_value = True From f53e239954a427fd3e37fb062c1d5a26a3ae261a Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 22 Apr 2015 16:58:18 -0400 Subject: [PATCH 29/35] add how to run tests to read me --- README.rst | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index bb06017..9b388b3 100644 --- a/README.rst +++ b/README.rst @@ -17,41 +17,37 @@ Installation **On Mac** -.. code-block:: console - - $: pip install landsat-util + ``$: pip install landsat-util`` **On Ubuntu 14.04** Use pip to install landsat-util. If you are not using virtualenv, you might have to run ``pip`` as ``sudo``. -.. code-block:: console - - $: sudo apt-get update - $: sudo apt-get install python-pip python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran - $: pip install landsat-util + ``$: sudo apt-get update`` + ``$: sudo apt-get install python-pip python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran`` + ``$: pip install landsat-util`` **On Other systems** -.. code-block:: console - - $: python setup.py install + ``$: python setup.py install`` **To Upgrade** -.. code-block:: console - - $: pip install -U landsat-util + ``$: pip install -U landsat-util`` If you have installed previous version of landsat using brew, first run: -.. code-block:: console + ``$: brew uninstall landsat-util`` + +**To Test** - $: brew uninstall landsat-util + ``$: pip install -U requirements/dev.txt`` + ``$: nosetests`` Overview: What can landsat-util do? -============ +==================================== + Landsat-util has three main functions: - **Search** for landsat tiles based on several search parameters. @@ -63,7 +59,7 @@ These three functions have to be performed separately. **Help**: Type ``landsat -h`` for detailed usage parameters. Step 1: Search -============ +=============== Search returns information about all landsat tiles that match your criteria. This includes a link to an unprocessed preview of the tile. The most important result is the tile's *sceneID*, which you will need to download the tile (see step 2 below). @@ -89,7 +85,7 @@ Search by latitude and longitude: Step 2: Download -============ +================= You can download tiles using their unique sceneID, which you get from landsat search. @@ -110,7 +106,7 @@ Download multiple sceneIDs: ``$: landsat download LC80090452014008LGN00 LC80090452015008LGN00 LC80090452013008LGN00`` Step 3: Image processing -============ +========================= You can process your downloaded tiles with our custom image processing algorithms. In addition, you can choose to pansharpen your images and specify which bands to process. @@ -145,7 +141,7 @@ Important Notes - Landsat-util requires at least 2GB of Memory (RAM). Recently Added -++++++++++ ++++++++++++++++ - Add longitude latitude search - Improve console output From 9ac08bcdaa0d330c125c975b35ef8a04947d7321 Mon Sep 17 00:00:00 2001 From: Scisco Date: Wed, 22 Apr 2015 17:00:25 -0400 Subject: [PATCH 30/35] bump up version to 0.6.0 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index e0ca4de..6289a7b 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -4,4 +4,4 @@ if not settings.DEBUG: sys.tracebacklimit = 0 -__version__ = '0.5.5' +__version__ = '0.6.0' From c935d2f1fe8fdf51a312bc8209e47577f64fcb54 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 23 Apr 2015 11:25:55 -0400 Subject: [PATCH 31/35] installation with python setup --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 9b388b3..70f6cfe 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,7 @@ Use pip to install landsat-util. If you are not using virtualenv, you might have **On Other systems** + ``$: python setup.py numpy six`` ``$: python setup.py install`` From a613a1c05392f915d2fc570a2e7073106f068095 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 23 Apr 2015 11:26:19 -0400 Subject: [PATCH 32/35] use updated version of gdal for updates (fix travis test fail) --- .travis.yml | 3 ++- Vagrantfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3499f55..681086b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,8 @@ before_install: # Install packages install: - - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib gdal scikit-image requests six nose dateutil + - conda install --yes python=$TRAVIS_PYTHON_VERSION -c https://conda.binstar.org/osgeo gdal + - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib scikit-image requests six nose dateutil - pip install --install-option="--no-cython-compile" cython - pip install -r requirements/travis.txt diff --git a/Vagrantfile b/Vagrantfile index 172dd6e..cc5949d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -25,7 +25,8 @@ Vagrant.configure(2) do |config| sudo rm -rf /dev/shm sudo ln -s /run/shm /dev/shm - conda install --yes numpy scikit-image requests gdal nose dateutil + conda install --yes -c https://conda.binstar.org/osgeo gdal + conda install --yes numpy scikit-image requests nose dateutil pip install --install-option="--no-cython-compile" cython pip install -r /vagrant/requirements/travis.txt From 89c0b8c0380329eececc0c00c7c0da833894e862 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 23 Apr 2015 13:19:29 -0400 Subject: [PATCH 33/35] update dep. to latest versions except request --- requirements/base.txt | 16 ++++++++-------- requirements/dev.txt | 12 ++++++------ requirements/travis.txt | 12 +++++------- setup.py | 15 ++++++++------- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 264f9c0..a17e798 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,10 +1,10 @@ requests==2.5.3 -python-dateutil==2.2 -termcolor==1.1.0 -numpy==1.9.1 -rasterio==0.18 +python-dateutil>=2.4.2 +termcolor>=1.1.0 +numpy>=1.9.2 +rasterio>=0.21.0 six==1.9.0 -scikit-image==0.10.1 -homura==0.1.0 -scipy==0.15.1 -boto==2.36.0 +scikit-image>=0.11.3 +homura>=0.1.1 +scipy>=0.15.1 +boto>=2.38.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index d0c7e23..05547bf 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,7 +1,7 @@ -r base.txt -pdoc==0.2.4 -nose==1.3.3 -coverage==3.7.1 -Sphinx==1.2.3 -wheel==0.23.0 -mock==1.0.1 +pdoc>=0.3.1 +nose>=1.3.6 +coverage>=3.7.1 +Sphinx>=1.3.1 +wheel>=0.24.0 +mock>=1.0.1 diff --git a/requirements/travis.txt b/requirements/travis.txt index 3d673b7..be8e7e3 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -1,7 +1,5 @@ -rasterio==0.18 -homura==0.1.0 -termcolor==1.1.0 -python-coveralls -mock==1.0.1 -boto==2.36.0 - +rasterio>=0.21.0 +homura>=0.1.1 +termcolor>=1.1.0 +python-dateutil>=2.4.2 +mock>=1.0.1lboto>=2.38.0 diff --git a/setup.py b/setup.py index d204637..0c477d0 100644 --- a/setup.py +++ b/setup.py @@ -35,14 +35,15 @@ def readme(): platforms='Posix; MacOS X; Windows', install_requires=[ 'requests==2.5.3', - 'python-dateutil==2.2', - 'numpy==1.9.1', - 'termcolor==1.1.0', - 'rasterio==0.18', + 'python-dateutil>=2.4.2', + 'numpy>=1.9.2', + 'termcolor>=1.1.0', + 'rasterio>=0.21.0', 'six==1.9.0', - 'scipy==0.15.1', - 'scikit-image==0.10.1', - 'homura==0.1.0' + 'scipy>=0.15.1', + 'scikit-image>=0.11.3', + 'homura>=0.1.1', + 'boto>=2.38.0' ], test_suite='nose.collector', test_require=test_requirements From 597f96693aa64e7cac0ea92001f024fb9af23d2c Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 23 Apr 2015 13:28:37 -0400 Subject: [PATCH 34/35] bump version to 0.6.1 --- landsat/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index 6289a7b..8411e55 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1,7 +1 @@ -import settings -import sys - -if not settings.DEBUG: - sys.tracebacklimit = 0 - -__version__ = '0.6.0' +__version__ = '0.6.1' From 182e2f1faab64f9751c4fa0684dd2809294560a6 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 23 Apr 2015 13:54:12 -0400 Subject: [PATCH 35/35] install gdal after installing python --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 681086b..66bb092 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,8 @@ before_install: # Install packages install: - - conda install --yes python=$TRAVIS_PYTHON_VERSION -c https://conda.binstar.org/osgeo gdal - conda install --yes python=$TRAVIS_PYTHON_VERSION numpy scipy matplotlib scikit-image requests six nose dateutil + - conda install --yes -c https://conda.binstar.org/osgeo gdal - pip install --install-option="--no-cython-compile" cython - pip install -r requirements/travis.txt