From 97e3ad193bc275dc05c96ad47883396bc9a04f8b Mon Sep 17 00:00:00 2001 From: Miles Watkins Date: Tue, 18 Aug 2015 01:46:56 -0400 Subject: [PATCH 01/50] Add street address search --- landsat/landsat.py | 10 ++++++++- landsat/search.py | 34 ++++++++++++++++++++++++----- landsat/utils.py | 47 +++++++++++++++++++++++++++++++++++++++++ requirements/base.txt | 1 + requirements/travis.txt | 1 + tests/test_utils.py | 5 +++++ 6 files changed, 92 insertions(+), 6 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 3a6a152..1fb5c50 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -29,7 +29,7 @@ Commands: Search: - landsat.py search [-p --pathrow] [--lat] [--lon] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] + landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] optional arguments: -p, --pathrow Paths and Rows in order separated by comma. Use quotes "001,003". @@ -39,6 +39,8 @@ --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 @@ -165,6 +167,7 @@ def args_options(): 'Example: path,row,path,row 001,001,190,204') parser_search.add_argument('--lat', type=float, help='The latitude') parser_search.add_argument('--lon', type=float, help='The longitude') + parser_search.add_argument('--address', type=str, help='The address') parser_download = subparsers.add_parser('download', help='Download images from Google Storage') @@ -268,9 +271,14 @@ def main(args): except ValueError: return ["The latitude and longitude values must be valid numbers", 1] + address = args.address + if address and (lat and lon): + return ["Cannot specify both address and latitude-longitude"] + result = s.search(paths_rows=args.pathrow, lat=lat, lon=lon, + address=address, limit=args.limit, start_date=args.start, end_date=args.end, diff --git a/landsat/search.py b/landsat/search.py index 65ebc46..5ee0be8 100644 --- a/landsat/search.py +++ b/landsat/search.py @@ -6,7 +6,7 @@ import requests import settings -from utils import three_digit, create_paired_list +from utils import three_digit, create_paired_list, geocode class Search(object): @@ -15,7 +15,7 @@ class Search(object): def __init__(self): self.api_url = settings.API_URL - def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, cloud_min=None, + def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None, limit=1): """ The main method of Search class. It searches Development Seed's Landsat API. @@ -32,6 +32,10 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -78,7 +82,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= } """ - search_string = self.query_builder(paths_rows, lat, lon, start_date, end_date, cloud_min, cloud_max) + search_string = self.query_builder(paths_rows, lat, lon, address, start_date, end_date, cloud_min, cloud_max) # Have to manually build the URI to bypass requests URI encoding # The api server doesn't accept encoded URIs @@ -109,7 +113,7 @@ def search(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date= return result - def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, end_date=None, + def query_builder(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None, cloud_max=None): """ Builds the proper search syntax (query) for Landsat API. @@ -125,6 +129,10 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en The The longitude :type lon: String, float, integer + :param address: + The address + :type address: + String :param start_date: Date string. format: YYYY-MM-DD :type start_date: @@ -171,7 +179,9 @@ def query_builder(self, paths_rows=None, lat=None, lon=None, start_date=None, en elif cloud_max: query.append(self.cloud_cover_prct_range_builder('-1', cloud_max)) - if lat and lon: + if address: + query.append(self.address_builder(address)) + elif lat and lon: query.append(self.lat_lon_builder(lat, lon)) if query: @@ -241,6 +251,20 @@ def cloud_cover_prct_range_builder(self, min=0, max=100): """ return 'cloudCoverFull:[%s+TO+%s]' % (min, max) + def address_builder(self, address): + """ Builds lat and lon query from a geocoded address. + + :param address: + The address + :type address: + String + + :returns: + String + """ + geocoded = geocode(address) + return self.lat_lon_builder(**geocoded) + def lat_lon_builder(self, lat=0, lon=0): """ Builds lat and lon query. diff --git a/landsat/utils.py b/landsat/utils.py index 3002cb8..912475d 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -7,6 +7,7 @@ import re from cStringIO import StringIO from datetime import datetime +import geocoder from mixins import VerbosityMixin @@ -278,3 +279,49 @@ def convert_to_integer_list(value): except ValueError: pass return s + + +# Geocoding confidence scores, from https://github.com/DenisCarriere/geocoder/blob/master/docs/features/Confidence%20Score.md +geocode_confidences = { + 10: 0.25, + 9: 0.5, + 8: 1., + 7: 5., + 6: 7.5, + 5: 10., + 4: 15., + 3: 20., + 2: 25., + 1: 99999., + # 0: unable to locate at all +} + + +def geocode(address, required_precision_km=1.): + """ Identifies the coordinates of an address + + :param address: + the address to be geocoded + :type value: + String + :param required_precision_km: + the maximum permissible geographic uncertainty for the geocoding + :type required_precision_km: + float + + :returns: + dict + + :example: + >>> geocode('1600 Pennsylvania Ave NW, Washington, DC 20500') + {'lat': 38.89767579999999, 'lon': -77.0364827} + + """ + geocoded = geocoder.google(address) + precision_km = geocode_confidences[geocoded.confidence] + + if precision_km <= required_precision_km: + (lon, lat) = geocoded.geometry['coordinates'] + return {'lat': lat, 'lon': lon} + else: + raise ValueError("Address could not be precisely located") diff --git a/requirements/base.txt b/requirements/base.txt index a17e798..789f50c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,3 +8,4 @@ scikit-image>=0.11.3 homura>=0.1.1 scipy>=0.15.1 boto>=2.38.0 +geocoder>=1.5.1 diff --git a/requirements/travis.txt b/requirements/travis.txt index 2509138..b03a045 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -4,3 +4,4 @@ termcolor>=1.1.0 python-dateutil>=2.4.2 mock>=1.0.1 boto>=2.38.0 +geocoder>=1.5.1 diff --git a/tests/test_utils.py b/tests/test_utils.py index b42c28b..81af806 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -105,6 +105,11 @@ def test_convert_to_integer_list(self): r = utils.convert_to_integer_list('1,11,10,QA') self.assertEqual([1, 11, 10, 'QA'], r) + def test_geocode(self): + self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364827}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) + self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') + self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) + if __name__ == '__main__': unittest.main() From b7129b4e3aa67b975000658ae153ac33900b4b7a Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 23 Sep 2015 20:03:39 -0700 Subject: [PATCH 02/50] update join method for download URL replaced join() with "/".join() for building google and Amazon S3 URLs that are compatible across platforms. Remove trailing "/" from S3 and Google base URL in settings.py --- landsat/downloader.py | 4 ++-- landsat/settings.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index 10bbc6e..d4f9bf6 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -186,7 +186,7 @@ def google_storage_url(self, sat): (String) The URL to a google storage file """ filename = sat['scene'] + '.tar.bz' - return join(self.google, sat['sat'], sat['path'], sat['row'], filename) + return "/".join([self.google, sat['sat'], sat['path'], sat['row'], filename]) def amazon_s3_url(self, sat, filename): """ @@ -204,7 +204,7 @@ def amazon_s3_url(self, sat, filename): :returns: (String) The URL to a S3 file """ - return join(self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename) + return "/".join([self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename]) def remote_file_exists(self, url): """ Checks whether the remote file exists. diff --git a/landsat/settings.py b/landsat/settings.py index 3c55113..8030212 100644 --- a/landsat/settings.py +++ b/landsat/settings.py @@ -14,8 +14,8 @@ SATELLITE = 'L8' L8_METADATA_URL = 'http://landsat.usgs.gov/metadata_service/bulk_metadata_files/LANDSAT_8.csv' -GOOGLE_STORAGE = 'http://storage.googleapis.com/earthengine-public/landsat/' -S3_LANDSAT = 'http://landsat-pds.s3.amazonaws.com/' +GOOGLE_STORAGE = 'http://storage.googleapis.com/earthengine-public/landsat' +S3_LANDSAT = 'http://landsat-pds.s3.amazonaws.com' API_URL = 'https://api.developmentseed.org/landsat' # User's Home Directory From 8c68be8bf9706174731c57ab4479c868897c8f19 Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 27 Oct 2015 13:07:04 -0400 Subject: [PATCH 03/50] support bare json fixes #96 --- landsat/landsat.py | 15 +++++++++++++-- tests/test_landsat.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index c033423..01472ac 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -3,6 +3,7 @@ # Landsat Util # License: CC0 1.0 Universal +import sys import argparse import textwrap import json @@ -56,6 +57,8 @@ -c CLOUD, --cloud CLOUD Maximum cloud percentage. Default: 20 perct + --json Returns a bare JSON response + -h, --help Show this help message and exit Download: @@ -181,6 +184,7 @@ def args_options(): 'Example: path,row,path,row 001,001,190,204') parser_search.add_argument('--lat', type=float, help='The latitude') parser_search.add_argument('--lon', type=float, help='The longitude') + parser_search.add_argument('--json', action='store_true', help='Returns a bare JSON response') parser_download = subparsers.add_parser('download', help='Download images from Google Storage') @@ -313,6 +317,9 @@ def main(args): cloud_max=args.cloud) if result['status'] == 'SUCCESS': + if args.json: + return json.dumps(result) + if args.latest > 0: datelist = [] for i in range(0, result['total_returned']): @@ -431,8 +438,12 @@ def __main__(): global parser parser = args_options() args = parser.parse_args() - with timer(): - exit(*main(args)) + if args.json: + print main(args) + sys.exit(0) + else: + with timer(): + exit(*main(args)) if __name__ == "__main__": try: diff --git a/tests/test_landsat.py b/tests/test_landsat.py index 2c56d30..7e5dab2 100644 --- a/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -3,6 +3,7 @@ """Tests for landsat""" +import json import unittest import subprocess import errno @@ -67,6 +68,15 @@ def test_search_pr_wrong_input(self): self.assertEquals(landsat.main(self.parser.parse_args(args)), ['Check your request and try again', 1]) + def test_search_json_output(self): + """Test json output in search""" + args = ['search', '--latest', '10', '--json'] + + output = landsat.main(self.parser.parse_args(args)) + j = json.loads(output) + + self.assertEquals(type(j), dict) + @mock.patch('landsat.landsat.Downloader') def test_download_correct(self, mock_downloader): """Test download command with correct input""" From 973abe21490fc6d1491d902caea76664f6e0165e Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 27 Oct 2015 13:07:12 -0400 Subject: [PATCH 04/50] add bare json to docs --- docs/commands.rst | 2 ++ docs/overview.rst | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docs/commands.rst b/docs/commands.rst index bd97fdd..acdd84d 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -35,6 +35,8 @@ Commands -c CLOUD, --cloud CLOUD Maximum cloud percentage. Default: 20 perct + --json Returns a bare JSON response + -h, --help Show this help message and exit Download: diff --git a/docs/overview.rst b/docs/overview.rst index f952151..a3c71e3 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -36,6 +36,9 @@ Search by latitude and longitude:: $: landsat search --lat 38.9004204 --lon -77.0237117 +Search by latitude and longitude with pure json output:: + + $: landsat search --lat 38.9004204 --lon -77.0237117 --json Download ++++++++ From c8397889a33946ee3b8a3db7c23c7d03f483ab12 Mon Sep 17 00:00:00 2001 From: Miles Watkins Date: Tue, 27 Oct 2015 13:41:33 -0400 Subject: [PATCH 05/50] Tweak precision on White House address test --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index e148f03..66c9fdb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -106,7 +106,7 @@ def test_convert_to_integer_list(self): self.assertEqual([1, 11, 10, 'QA'], r) def test_geocode(self): - self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364827}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) + self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364823}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) From 260f74e34b489ad362157140945dc2220745c3c6 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 10:46:02 -0400 Subject: [PATCH 06/50] add test suits to the docker image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 77d498c..be6f2ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,4 @@ ADD landsat /usr/local/lib/python2.7/dist-packages/landsat ADD bin/landsat /usr/local/bin/ ADD . /landsat RUN cd /landsat && pip install -r requirements/docker.txt - +RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 From c8650d46827209186e853758246cf7a34a495570 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 10:46:09 -0400 Subject: [PATCH 07/50] use docker for testing --- .travis.yml | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7fc2963..4591991 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,44 +1,15 @@ language: python -sudo: false +sudo: required -cache: - directories: - - ~/.cache/pip - -env: - global: - - PIP_WHEEL_DIR=$HOME/.cache/pip/wheels - - PIP_FIND_LINKS=file://$HOME/.cache/pip/wheels - -virtualenv: - system_site_packages: true - -addons: - apt: - packages: - - libgdal1h - - gdal-bin - - libgdal-dev - - libatlas-dev - - libatlas-base-dev - - gfortran - - python-numpy - - python-scipy +services: + - docker python: - '2.7' -before_install: - - pip install -U pip - - pip install wheel - -install: - - "pip wheel -r requirements/dev.txt" - - "pip install -r requirements/dev.txt" - script: -- nosetests +- docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests deploy: provider: pypi From 5139fd691b28d273d9f7feab06fd72e06c90defe Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 10:46:16 -0400 Subject: [PATCH 08/50] add travis test note --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 123a4d9..9dbd1d8 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,11 @@ To run the documentation locally:: $ cd docs $ make html +Travis Tests +++++++++++++ + +To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag. + Recently Added Features +++++++++++++++++++++++ From c0ae5e8ae94f755775a0aa1229af217ca3fa4c7c Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:12:11 -0400 Subject: [PATCH 09/50] remove python --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4591991..686db65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,8 @@ -language: python - sudo: required services: - docker -python: -- '2.7' - script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests From 6a120142cbba99cf934a57d1f09dd9b484f1daa3 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:43:54 -0400 Subject: [PATCH 10/50] add docker test command --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 9dbd1d8..145df35 100644 --- a/README.rst +++ b/README.rst @@ -34,6 +34,12 @@ Travis Tests To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag. +To test with docker image locally run: + +.. code:: + + $ docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests + Recently Added Features +++++++++++++++++++++++ From 6cf52c17d156ee10a0a156e7cca659b50fe9f802 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:49:14 -0400 Subject: [PATCH 11/50] update todo list --- docs/todo.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/todo.rst b/docs/todo.rst index afb754d..704c1ab 100644 --- a/docs/todo.rst +++ b/docs/todo.rst @@ -1,9 +1,6 @@ To Do List ++++++++++ -- Add Sphinx Documentation -- Add capacity for NDVI output - Add alternative projections (currently only option is default web-mercator; EPSG: 3857) -- Connect search to Google Address API - Include 16-bit image variant in output - Add support for color correct looping over multiple compressed inputs (currently just 1) From ae22ea3d3226a8bb318bd624a79366f5bd5c7a1f Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:49:21 -0400 Subject: [PATCH 12/50] update recently added features --- README.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 145df35..f54cac1 100644 --- a/README.rst +++ b/README.rst @@ -43,9 +43,10 @@ To test with docker image locally run: Recently Added Features +++++++++++++++++++++++ -- Add longitude latitude search -- Improve console output -- Add more color options such as false color, true color, etc. +- Improved pansharpening +- Use BQA bands for cloud/snow coverage and use in color correction +- Add support for different NDVI color maps (three included) +- Add support for image clipping using the new `--clip` flag Change Log ++++++++++ From 499ab1c24ccc9b645622c68f71fbb64a265594f7 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:53:58 -0400 Subject: [PATCH 13/50] update link to docs [ci skip] --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f54cac1..0f15ce9 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ Landsat-util is a command line utility that makes it easy to search, download, a Docs +++++ -For full documentation visit: http://landsat-util.readthedocs.org/ +For full documentation visit: https://pythonhosted.org/landsat-util/ To run the documentation locally:: From fad40e25132ca571007bdae942df1ec489b427a5 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 11:55:29 -0400 Subject: [PATCH 14/50] update commands [ci skip] --- docs/commands.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/commands.rst b/docs/commands.rst index acdd84d..c2de47c 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -20,6 +20,8 @@ Commands --lon Longitude + --address Street address + -l LIMIT, --limit LIMIT Search return results limit default is 10 From f4fa584d3fd052ba705aa0d7df6e5ee1118cc6ee Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:00:22 -0400 Subject: [PATCH 15/50] build docker image first --- .travis.yml | 3 +++ README.rst | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 686db65..ad7d4a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ sudo: required services: - docker +before_script: +- docker build -t 'developmentseed/landsat-util:travis' . + script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests diff --git a/README.rst b/README.rst index 0f15ce9..150181c 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ To run the documentation locally:: Travis Tests ++++++++++++ -To speed up testing on travis, we use a docker image. If you add new packages, to ensure that tests will pass on Travis, make sure to rebuild the docker image and push it to DockerHub with `travis` tag. +To speed up testing on travis, we use a docker image. To test with docker image locally run: From 579a339bbef2bfbefe9f3577a10b2f00f107850b Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:25:02 -0400 Subject: [PATCH 16/50] use special dockerfile for travis --- .travis.yml | 3 ++- Dockerfile | 1 - travis-dockerfile | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 travis-dockerfile diff --git a/.travis.yml b/.travis.yml index ad7d4a9..8e9fd05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ services: - docker before_script: -- docker build -t 'developmentseed/landsat-util:travis' . +- docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests @@ -26,3 +26,4 @@ after_deploy: docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker push developmentseed/landsat-util:$VER fi + docker push developmentseed/landsat-util:travis diff --git a/Dockerfile b/Dockerfile index be6f2ef..8bf1789 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,3 @@ ADD landsat /usr/local/lib/python2.7/dist-packages/landsat ADD bin/landsat /usr/local/bin/ ADD . /landsat RUN cd /landsat && pip install -r requirements/docker.txt -RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 diff --git a/travis-dockerfile b/travis-dockerfile new file mode 100644 index 0000000..92d8fa1 --- /dev/null +++ b/travis-dockerfile @@ -0,0 +1,4 @@ +FROM developmentseed/landsat-util:dev +ADD . /test +RUN cd /test && pip install -r requirements/docker.txt +RUN pip install pdoc>=0.3.1 nose>=1.3.7 coverage>=4.0 Sphinx>=1.3.1 wheel>=0.26.0 mock>=1.3.0 From 0a936a63249acc1442fa78bd7c1c214d9ea25392 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:25:46 -0400 Subject: [PATCH 17/50] always login to docker --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8e9fd05..e10d4b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ deploy: branch: master after_deploy: + docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" VER=$(python -c "import landsat; print landsat.__version__") docker build . -t developmentseed/landsat-util:$VER - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker push developmentseed/landsat-util:$VER fi docker push developmentseed/landsat-util:travis From 2bfb4aac5f34ed2541eb5cdeeb678947f60f7a6d Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:31:51 -0400 Subject: [PATCH 18/50] push to docker after success --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e10d4b3..9f3fa3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,10 @@ before_script: script: - docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests +after_success: +- docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} +- docker push developmentseed/landsat-util:travis + deploy: provider: pypi user: devseed @@ -19,11 +23,11 @@ deploy: branch: master after_deploy: - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" VER=$(python -c "import landsat; print landsat.__version__") + docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} docker build . -t developmentseed/landsat-util:$VER docker push developmentseed/landsat-util:$VER fi - docker push developmentseed/landsat-util:travis + From 19369f370b147cfd5fe0501c874e6b414e57bc94 Mon Sep 17 00:00:00 2001 From: Scisco Date: Thu, 29 Oct 2015 12:50:33 -0400 Subject: [PATCH 19/50] add docker hub login info --- .travis.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f3fa3b..0692e6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,20 @@ sudo: required services: - - docker +- docker + +env: + global: + - secure: QsF7ignSAbH/WCyO6v9bw1exmCWDQR0DqmHkwJ5swc9N44OOOzbWGsaMSYB5y9h+d70fz4arbxQDhsk2KvX4Zd1/2YIMOrIsbgDYeegpkhVPgyQNPKmVqiX+Tb47t1C/TgkC7A07tiPpuefYcLNMZ8gzz7oKhh1UKapYftqzZ+g= + - secure: HxjeKWSROBQYy9NuNkgQeaK1ubTF8vH5FcR8nUTSAYxxw/qOzKpqkiq4BcJSRcIwTbkvaBf4MshLGVOxPjMeyJFe06UD/6LvTUGS3bwdya+m0RFjHe5/3wzS8/MxLbTlvgzmuGLLKOsJjXCi9eQQchKfHv+QuhGxhYVLQpnbU9E= + - secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo= + before_script: - docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: -- docker run --rm -it -v "$(pwd)":/test developmentseed/landsat-util:travis nosetests +- docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis nosetests after_success: - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} @@ -25,9 +32,7 @@ deploy: after_deploy: if [ "$TRAVIS_BRANCH" == "master" ]; then echo "Start Docker Hub Push" - VER=$(python -c "import landsat; print landsat.__version__") - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} + VER=$(docker run --rm -it -v "$(pwd)":/test -w /test developmentseed/landsat-util:travis python landsat/landsat.py --version | sed s/[^0-9\.]//g) docker build . -t developmentseed/landsat-util:$VER docker push developmentseed/landsat-util:$VER fi - From 6bcdd083263844bce9e9addf6cd1556bc8e8a308 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 28 Nov 2015 18:59:37 -0800 Subject: [PATCH 20/50] Correct default for download `landsat download LC80450332015023LGN00` should download the tar.gz file with all bands by default, but the default is being set to '432'. This does not do anything for scenes prior to 2015 on Google Storage, but for 2015 on AWS it only downloads those bands. This is not expected behaviour according to the docs, and it makes it hard to download the whole tar archive (you need to use `landsat download LC80450332015023LGN00 -b ""`) --- landsat/landsat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 376ac2f..167e56a 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -197,7 +197,7 @@ def args_options(): help="Provide Full sceneID, e.g. LC81660392014196LGN00") parser_download.add_argument('-b', '--bands', help='If you specify bands, landsat-util will try to download ' - 'the band from S3. If the band does not exist, an error is returned', default='432') + 'the band from S3. If the band does not exist, an error is returned', default='') parser_download.add_argument('-d', '--dest', help='Destination path') parser_download.add_argument('-p', '--process', help='Process the image after download', action='store_true') parser_download.add_argument('--pansharpen', action='store_true', From 16f4d232caac117a3d26f069817e306fbf473140 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 15:17:19 -0500 Subject: [PATCH 21/50] linting --- landsat/landsat.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 376ac2f..1f0e2e0 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -32,7 +32,8 @@ Commands: Search: - landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] [-h] + landsat.py search [-p --pathrow] [--lat] [--lon] [--address] [-l LIMIT] [-s START] [-e END] [-c CLOUD] + [-h] optional arguments: -p, --pathrow Paths and Rows in order separated by comma. Use quotes "001,003". @@ -88,7 +89,8 @@ --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. - Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625,50.2682767372753 + Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625, + 50.2682767372753 -u --upload Upload to S3 after the image processing completed @@ -124,7 +126,8 @@ --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. - Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625,50.2682767372753 + Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625, + 50.2682767372753 -v, --verbose Show verbose output @@ -297,7 +300,7 @@ def main(args): if args.latest > 0: args.limit = 25 end = datetime.now() - start = end-relativedelta(days=+365) + start = end - relativedelta(days=+365) args.end = end.strftime("%Y-%m-%d") args.start = start.strftime("%Y-%m-%d") except (TypeError, ValueError): From fbd7458bae7392e573a6e6c7d90ce5754984bfcd Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 15:50:26 -0500 Subject: [PATCH 22/50] apply ndvigrey to process and download --- landsat/landsat.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index 1f0e2e0..bf75cae 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -87,6 +87,8 @@ --ndvi Calculates NDVI and produce a RGB GTiff with seperate colorbar. + --ndvigrey Calculates NDVI and produce a greyscale GTiff. + --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625, @@ -380,7 +382,7 @@ def main(args): path = path + '.tar.bz' stored = process_image(path, args.bands, False, args.pansharpen, args.ndvi, force_unzip, - bounds=bounds) + args.ndvigrey, bounds=bounds) if args.upload: try: @@ -449,9 +451,10 @@ def __main__(): global parser parser = args_options() args = parser.parse_args() - if args.json: - print main(args) - sys.exit(0) + if args.subs == 'search': + if args.json: + print main(args) + sys.exit(0) else: with timer(): exit(*main(args)) From 5af7dd3c3355a214d54f30b76ea1c6e16cbea077 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 15:50:57 -0500 Subject: [PATCH 23/50] don't apply colormap to ndvi fixes #127 --- landsat/ndvi.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/landsat/ndvi.py b/landsat/ndvi.py index edda4ac..931192c 100644 --- a/landsat/ndvi.py +++ b/landsat/ndvi.py @@ -88,8 +88,6 @@ def write_band(self, output_band, output_file, image_data): output.write_band(1, output_band) - cmap = {k: v[:3] for k, v in self.cmap.iteritems()} - output.write_colormap(1, cmap) self.output("Writing to file", normal=True, color='green', indent=1) return output_file @@ -123,7 +121,7 @@ def write_band(self, output_band, output_file, image_data): crs=self.dst_crs) as output: for i in range(3): - output.write_band(i+1, rgb_bands[i]) + output.write_band(i + 1, rgb_bands[i]) self.output("Writing to file", normal=True, color='green', indent=1) return output_file From d12894431acb8ae8f142e65cd35806599dd820df Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 15:55:29 -0500 Subject: [PATCH 24/50] add ndvigrey to download --- landsat/landsat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/landsat/landsat.py b/landsat/landsat.py index bf75cae..b0e45e6 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -210,6 +210,7 @@ def args_options(): 'image. Pansharpening requires larger memory') parser_download.add_argument('--ndvi', action='store_true', help='Whether to run the NDVI process. If used, bands parameter is disregarded') + parser_download.add_argument('--ndvigrey', action='store_true', help='Create an NDVI map in grayscale (grey)') parser_download.add_argument('--clip', help='Clip the image with the bounding box provided. Values must be in ' + 'WGS84 datum, and with longitude and latitude units of decimal degrees ' + 'separated by comma.' + From 59a6190ab45534eddedb6878b1545f2a3f74dc7a Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 16:00:09 -0500 Subject: [PATCH 25/50] download 4, 5 if ndvigrey is used --- landsat/landsat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index b0e45e6..46755bc 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -365,7 +365,7 @@ def main(args): if args.pansharpen: bands.append(8) - if args.ndvi: + if args.ndvi or args.ndvigrey: bands = [4, 5] downloaded = d.download(args.scenes, bands) From 6583794a8341ef04c14b4dc40011ed6086954875 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 16:29:09 -0500 Subject: [PATCH 26/50] adjust for WH coordinates change --- tests/test_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 66c9fdb..bffd67c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -106,9 +106,13 @@ def test_convert_to_integer_list(self): self.assertEqual([1, 11, 10, 'QA'], r) def test_geocode(self): - self.assertEqual({'lat': 38.89767579999999, 'lon': -77.0364823}, utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500')) + loc = utils.geocode('1600 Pennsylvania Ave NW, Washington, DC 20500') + + self.assertEqual(round(loc['lat'], 3), 38.898) + self.assertEqual(round(loc['lon'], 3), -77.037) self.assertRaises(ValueError, utils.geocode, 'Pennsylvania Ave NW, Washington, DC') - self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) + self.assertEqual({'lat': 38.8987352, 'lon': -77.0350902}, + utils.geocode('Pennsylvania Ave NW, Washington, DC', 10.)) def test_convert_to_float_list(self): # correct input From bf1e35d38bddb75a36c5a9ffc1856e449e12c1f7 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 16:29:20 -0500 Subject: [PATCH 27/50] capture change in function call --- tests/test_landsat.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_landsat.py b/tests/test_landsat.py index 7e5dab2..06ad50f 100644 --- a/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -107,7 +107,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', - False, False, False, False, bounds=None) + False, False, False, False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with force unzip flag @@ -116,7 +116,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, - True, bounds=None) + True, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with pansharpen @@ -125,7 +125,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, - False, bounds=None) + False, False, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with pansharpen and clipping @@ -134,7 +134,7 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 3, 2, 8]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, True, False, - False, bounds=[-180.0, -180.0, 0.0, 0.0]) + False, False, bounds=[-180.0, -180.0, 0.0, 0.0]) self.assertEquals(output, ["The output is stored at image.TIF", 0]) # Call with ndvi @@ -143,7 +143,16 @@ def test_download_process_continuous(self, mock_downloader, mock_process): output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True, - False, bounds=None) + False, False, bounds=None) + self.assertEquals(output, ["The output is stored at image.TIF", 0]) + + # Call with ndvigrey + args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', + self.mock_path, '-p', '--ndvigrey'] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], [4, 5]) + mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False, + False, True, bounds=None) self.assertEquals(output, ["The output is stored at image.TIF", 0]) @mock.patch('landsat.landsat.Uploader') @@ -160,7 +169,7 @@ def test_download_process_continuous_with_upload(self, mock_downloader, mock_pro output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, - False, bounds=None) + False, False, bounds=None) mock_upload.assert_called_with('somekey', 'somesecret', 'this') mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF') self.assertEquals(output, ['The output is stored at image.TIF', 0]) @@ -177,7 +186,7 @@ def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock output = landsat.main(self.parser.parse_args(args)) mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False, - False, bounds=None) + False, False, bounds=None) self.assertEquals(output, ['Could not authenticate with AWS', 1]) @mock.patch('landsat.landsat.process_image') From 18cbdba4f30968760f86bfb00d80a810e83f8b22 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 16:31:02 -0500 Subject: [PATCH 28/50] push to pypi on both master and develop branches --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0692e6b..5c53fbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,9 @@ deploy: secure: WtawFW/999XYszmZfj1Qk82l00OSyP2JBVOOGCERrW1gVO7MYtYsgP31HKRSzNTCTHJNVDpdK4WZWY6zPQqC3l2UfWYYsvRn0hCoI8AJxE5VCUEg6Ccpe6fMJuhp1pq6Zy7yrfBSZcOB9aqSHLBJsunD2o3mNlTC8WV8vNK74ck= on: repo: developmentseed/landsat-util - branch: master + branch: + - master + - develop after_deploy: if [ "$TRAVIS_BRANCH" == "master" ]; then From 1333af285046092fa11903a69919e41746e123ff Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 16:34:21 -0500 Subject: [PATCH 29/50] bump version to 0.10b1 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index 8969d49..6596629 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.9.1' +__version__ = '0.10b1' From e1aace24127de05564facd9092b3b33fe52fbf9f Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:06:02 -0500 Subject: [PATCH 30/50] if process requested after download use 432 --- landsat/landsat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/landsat/landsat.py b/landsat/landsat.py index 97f6c40..ea72ff3 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -371,6 +371,8 @@ def main(args): downloaded = d.download(args.scenes, bands) if args.process: + if not args.bands: + args.bands = '432' force_unzip = True if args.force_unzip else False for scene, src in downloaded.iteritems(): if args.dest: From 1e2fc64117e37e9adf548a2f6a551983f82ceee4 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:06:09 -0500 Subject: [PATCH 31/50] test default download --- tests/test_landsat.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_landsat.py b/tests/test_landsat.py index 06ad50f..c517068 100644 --- a/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -88,6 +88,17 @@ def test_download_correct(self, mock_downloader): mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], [11]) self.assertEquals(output, ['Download Completed', 0]) + @mock.patch('landsat.landsat.Downloader') + def test_download_correct_zip(self, mock_downloader): + """Download command should download zip if no bands are given""" + mock_downloader.download.return_value = True + + args = ['download', 'LC80010092015051LGN00', '-d', self.mock_path] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(download_dir=self.mock_path) + mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], []) + self.assertEquals(output, ['Download Completed', 0]) + def test_download_incorrect(self): """Test download command with incorrect input""" args = ['download', 'LT813600'] From 7413e4a0a28d6d7c9b71389d4006c4ce664ddc0b Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:06:34 -0500 Subject: [PATCH 32/50] update commands help file --- docs/commands.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/commands.rst b/docs/commands.rst index c2de47c..c2daeb1 100644 --- a/docs/commands.rst +++ b/docs/commands.rst @@ -64,6 +64,8 @@ Commands --ndvi Calculates NDVI and produce a RGB GTiff with seperate colorbar. + --ndvigrey Calculates NDVI and produce a greyscale GTiff. + --clip Clip the image with the bounding box provided. Values must be in WGS84 datum, and with longitude and latitude units of decimal degrees separated by comma. Example: --clip -346.06658935546875,49.93531194616915,-345.4595947265625,50.2682767372753 From d1efe8fe3e5df712e9a362c2dac87621326e593a Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:08:31 -0500 Subject: [PATCH 33/50] update overview guide --- docs/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/overview.rst b/docs/overview.rst index a3c71e3..43c7c4c 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -53,7 +53,7 @@ Download images by their custom sceneID, which you get from landsat search:: $: landsat download LC80090452014008LGN00 -Download only band 4, 3 and 2 for a particular sceneID:: +By default landsat-util downloads the full zip file from Google Storage unless you specify the bands or run an image processing right after download. For example to download only band 4, 3 and 2 for a particular sceneID run:: $: landsat download LC80090452014008LGN00 --bands 432 From fb0ccc6b7a3d3f299d719f2309cd9996f8b7c40b Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:27:49 -0500 Subject: [PATCH 34/50] apply pansharpen and ndvi only if -p is used --- landsat/landsat.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/landsat/landsat.py b/landsat/landsat.py index ea72ff3..fb87f1e 100755 --- a/landsat/landsat.py +++ b/landsat/landsat.py @@ -362,11 +362,16 @@ def main(args): d = Downloader(download_dir=args.dest) try: bands = convert_to_integer_list(args.bands) - if args.pansharpen: - bands.append(8) - if args.ndvi or args.ndvigrey: - bands = [4, 5] + if args.process: + if args.pansharpen: + bands.append(8) + + if args.ndvi or args.ndvigrey: + bands = [4, 5] + + if not args.bands: + bands = [4, 3, 2] downloaded = d.download(args.scenes, bands) From 2862d4c3e4c03f32799d81e81c5d202548b70f0e Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:28:00 -0500 Subject: [PATCH 35/50] test default bands are applied with -p flag --- tests/test_landsat.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_landsat.py b/tests/test_landsat.py index c517068..e7667e4 100644 --- a/tests/test_landsat.py +++ b/tests/test_landsat.py @@ -99,6 +99,18 @@ def test_download_correct_zip(self, mock_downloader): mock_downloader.return_value.download.assert_called_with(['LC80010092015051LGN00'], []) self.assertEquals(output, ['Download Completed', 0]) + @mock.patch('landsat.landsat.process_image') + @mock.patch('landsat.landsat.Downloader.download') + def test_download_no_bands_with_process(self, mock_downloader, mock_process): + """Download command should not download zip if no bands are given but process flag is used""" + mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'} + mock_process.return_value = 'image.TIF' + + args = ['download', 'LC80010092015051LGN00', '-p', '-d', self.mock_path] + output = landsat.main(self.parser.parse_args(args)) + mock_downloader.assert_called_with(['LC80010092015051LGN00'], [4, 3, 2]) + self.assertEquals(output, ["The output is stored at image.TIF", 0]) + def test_download_incorrect(self): """Test download command with incorrect input""" args = ['download', 'LT813600'] From 9186ec2bd8da4cf59e7e7f6ba61132cdf5544c6d Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:41:19 -0500 Subject: [PATCH 36/50] add geocoder --- setup.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index e4afc11..27c50b8 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,8 @@ def readme(): return f.read() test_requirements = [ - 'nose>=1.3.7', - 'mock>=1.3.0' + 'nose>=1.3.7', + 'mock>=1.3.0' ] setup( @@ -35,17 +35,18 @@ def readme(): license='CCO', platforms='Posix; MacOS X; Windows', install_requires=[ - 'requests==2.7.0', - 'python-dateutil>=2.4.2', - 'numpy>=1.9.3', - 'termcolor>=1.1.0', - 'rasterio>=0.26.0', - 'six==1.9.0', - 'scipy>=0.16.0', - 'scikit-image>=0.11.3', - 'homura>=0.1.2', - 'boto>=2.38.0', - 'polyline==1.1' + 'requests==2.7.0', + 'python-dateutil>=2.4.2', + 'numpy>=1.9.3', + 'termcolor>=1.1.0', + 'rasterio>=0.26.0', + 'six==1.9.0', + 'scipy>=0.16.0', + 'scikit-image>=0.11.3', + 'homura>=0.1.2', + 'boto>=2.38.0', + 'polyline==1.1', + 'geocoder>=1.5.1' ], test_suite='nose.collector', tests_require=test_requirements From 4cb91431559d487ed77d830e66ed0ef38e5d0435 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 17:41:34 -0500 Subject: [PATCH 37/50] bump up version to 0.10b2 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index 6596629..6dd189b 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.10b1' +__version__ = '0.10b2' From 9ee59fd4f5ea2de7031cdaf8df63ff6df44677f8 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:04:43 -0500 Subject: [PATCH 38/50] a new url builder --- landsat/utils.py | 14 ++++++++++++++ tests/test_utils.py | 10 ++++++++++ 2 files changed, 24 insertions(+) diff --git a/landsat/utils.py b/landsat/utils.py index 840ea91..c6a3545 100644 --- a/landsat/utils.py +++ b/landsat/utils.py @@ -391,3 +391,17 @@ def adjust_bounding_box(bounds1, bounds2): new_bounds[3] = bounds1[3] return tuple(new_bounds) + + +def remove_slash(value): + + assert(isinstance(value, str)) + return re.sub('(^\/|\/$)', '', value) + + +def url_builder(segments): + + # Only accept list or tuple + assert((isinstance(segments, list) or isinstance(segments, tuple))) + return "/".join([remove_slash(s) for s in segments]) + diff --git a/tests/test_utils.py b/tests/test_utils.py index bffd67c..9bbd38d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -175,6 +175,16 @@ def test_adjust_bounding_box(self): self.assertEqual(utils.adjust_bounding_box(origin, target), origin) + def test_url_builder(self): + + self.assertEqual('http://example.com/segment1/segment2', + utils.url_builder(['/http://example.com', 'segment1/', '/segment2'])) + + self.assertEqual('http://example.com/segment1/segment2', + utils.url_builder(('/http://example.com', 'segment1/', '/segment2',))) + + with self.assertRaises(AssertionError): + utils.url_builder('example.com') if __name__ == '__main__': unittest.main() From 15ac13787bab8e3b13b57a575c3ea46a11e1ec3c Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:04:51 -0500 Subject: [PATCH 39/50] keep slashes --- landsat/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/landsat/settings.py b/landsat/settings.py index 6dd8400..331f507 100644 --- a/landsat/settings.py +++ b/landsat/settings.py @@ -14,8 +14,8 @@ SATELLITE = 'L8' L8_METADATA_URL = 'http://landsat.usgs.gov/metadata_service/bulk_metadata_files/LANDSAT_8.csv' -GOOGLE_STORAGE = 'http://storage.googleapis.com/earthengine-public/landsat' -S3_LANDSAT = 'http://landsat-pds.s3.amazonaws.com' +GOOGLE_STORAGE = 'http://storage.googleapis.com/earthengine-public/landsat/' +S3_LANDSAT = 'http://landsat-pds.s3.amazonaws.com/' API_URL = 'https://api.developmentseed.org/landsat' # User's Home Directory From 47051d003a95ba5da2eb8efbc58fcb26ab24faeb Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:05:08 -0500 Subject: [PATCH 40/50] use url_builder instead of join --- landsat/downloader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/landsat/downloader.py b/landsat/downloader.py index 81b119b..7641298 100644 --- a/landsat/downloader.py +++ b/landsat/downloader.py @@ -5,7 +5,7 @@ from homura import download as fetch import requests -from utils import check_create_folder +from utils import check_create_folder, url_builder from mixins import VerbosityMixin import settings @@ -192,7 +192,7 @@ def google_storage_url(self, sat): (String) The URL to a google storage file """ filename = sat['scene'] + '.tar.bz' - return "/".join([self.google, sat['sat'], sat['path'], sat['row'], filename]) + return url_builder([self.google, sat['sat'], sat['path'], sat['row'], filename]) def amazon_s3_url(self, sat, filename): """ @@ -210,7 +210,7 @@ def amazon_s3_url(self, sat, filename): :returns: (String) The URL to a S3 file """ - return "/".join([self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename]) + return url_builder([self.s3, sat['sat'], sat['path'], sat['row'], sat['scene'], filename]) def remote_file_exists(self, url): """ Checks whether the remote file exists. @@ -284,4 +284,4 @@ def scene_interpreter(self, scene): d = Downloader() - # d.download(['LC81990242015046LGN00', 'LC80030172015001LGN00']) \ No newline at end of file + # d.download(['LC81990242015046LGN00', 'LC80030172015001LGN00']) From d4dc657e777370f3b361596e42dc3f720fdbe698 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:05:27 -0500 Subject: [PATCH 41/50] install twine before travis deploy --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5c53fbf..f3e7166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ after_success: - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} - docker push developmentseed/landsat-util:travis +before_deploy: + - pip install twine + deploy: provider: pypi user: devseed From d04f69b75f9395f27ce5339766afb79f2922017d Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:08:00 -0500 Subject: [PATCH 42/50] bump to 0.10b3 --- landsat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/landsat/__init__.py b/landsat/__init__.py index 6dd189b..ca16de9 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.10b2' +__version__ = '0.10b3' From 91f7e70fc993c89db35fb990c42cf854589a94d2 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:23:36 -0500 Subject: [PATCH 43/50] install twine in the beginning --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f3e7166..20964d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: before_script: +- pip install twine - docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: @@ -20,9 +21,6 @@ after_success: - docker login -e ${DOCKER_EMAIL} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} - docker push developmentseed/landsat-util:travis -before_deploy: - - pip install twine - deploy: provider: pypi user: devseed From dff5a8a1b95ad8bd7f8cfd4c3d3dec6328a87f32 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:26:48 -0500 Subject: [PATCH 44/50] use edge --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 20964d1..93c627e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ env: before_script: -- pip install twine - docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . script: @@ -24,6 +23,7 @@ after_success: deploy: provider: pypi user: devseed + edge: true password: secure: WtawFW/999XYszmZfj1Qk82l00OSyP2JBVOOGCERrW1gVO7MYtYsgP31HKRSzNTCTHJNVDpdK4WZWY6zPQqC3l2UfWYYsvRn0hCoI8AJxE5VCUEg6Ccpe6fMJuhp1pq6Zy7yrfBSZcOB9aqSHLBJsunD2o3mNlTC8WV8vNK74ck= on: From 7c289bc91ae23ccae4bef39acb0e40e2e3987e20 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:37:08 -0500 Subject: [PATCH 45/50] install twine --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 93c627e..402e226 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,9 @@ env: - secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo= +install: +- pip install twine + before_script: - docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . @@ -23,7 +26,6 @@ after_success: deploy: provider: pypi user: devseed - edge: true password: secure: WtawFW/999XYszmZfj1Qk82l00OSyP2JBVOOGCERrW1gVO7MYtYsgP31HKRSzNTCTHJNVDpdK4WZWY6zPQqC3l2UfWYYsvRn0hCoI8AJxE5VCUEg6Ccpe6fMJuhp1pq6Zy7yrfBSZcOB9aqSHLBJsunD2o3mNlTC8WV8vNK74ck= on: From 2ef77bab038d4a58f2a058b56801893bf6459408 Mon Sep 17 00:00:00 2001 From: scisco Date: Tue, 1 Dec 2015 19:39:25 -0500 Subject: [PATCH 46/50] with user --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 402e226..ff65cec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: install: -- pip install twine +- pip install --user twine before_script: - docker build --file="travis-dockerfile" -t "developmentseed/landsat-util:travis" . From 5e9034115fb85f515be8544fdc8eb3cc76805fca Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 5 Jan 2016 14:30:25 -0500 Subject: [PATCH 47/50] second attempt in fixing #114 --- landsat/maps/colormap_ndvi_cfastie.txt | 192 ++++++++++++------------- 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/landsat/maps/colormap_ndvi_cfastie.txt b/landsat/maps/colormap_ndvi_cfastie.txt index 095f1d9..5814e53 100644 --- a/landsat/maps/colormap_ndvi_cfastie.txt +++ b/landsat/maps/colormap_ndvi_cfastie.txt @@ -165,99 +165,99 @@ mode = 255 26 233 47 17 240 31 8 247 15 -0 255 0 -7 255 0 -15 255 0 -23 255 0 -31 255 0 -39 255 0 -47 255 0 -55 255 0 -63 255 0 -71 255 0 -79 255 0 -87 255 0 -95 255 0 -103 255 0 -111 255 0 -119 255 0 -127 255 0 -135 255 0 -143 255 0 -151 255 0 -159 255 0 -167 255 0 -175 255 0 -183 255 0 -191 255 0 -199 255 0 -207 255 0 -215 255 0 -223 255 0 -231 255 0 -239 255 0 -247 255 0 -255 255 0 -255 249 0 -255 244 0 -255 239 0 -255 233 0 -255 228 0 -255 223 0 -255 217 0 -255 212 0 -255 207 0 -255 201 0 -255 196 0 -255 191 0 -255 185 0 -255 180 0 -255 175 0 -255 170 0 -255 164 0 -255 159 0 -255 154 0 -255 148 0 -255 143 0 -255 138 0 -255 132 0 -255 127 0 -255 122 0 -255 116 0 -255 111 0 -255 106 0 -255 100 0 -255 95 0 -255 90 0 -255 85 0 -255 79 0 -255 74 0 -255 69 0 -255 63 0 -255 58 0 -255 53 0 -255 47 0 -255 42 0 -255 37 0 -255 31 0 -255 26 0 -255 21 0 -255 15 0 -255 10 0 -255 5 0 -255 0 0 -255 0 15 -255 0 31 -255 0 47 -255 0 63 -255 0 79 -255 0 95 -255 0 111 -255 0 127 -255 0 143 -255 0 159 -255 0 175 -255 0 191 -255 0 207 -255 0 223 -255 0 239 +1 255 1 +7 255 1 +15 255 1 +23 255 1 +31 255 1 +39 255 1 +47 255 1 +55 255 1 +63 255 1 +71 255 1 +79 255 1 +87 255 1 +95 255 1 +103 255 1 +111 255 1 +119 255 1 +127 255 1 +135 255 1 +143 255 1 +151 255 1 +159 255 1 +167 255 1 +175 255 1 +183 255 1 +191 255 1 +199 255 1 +207 255 1 +215 255 1 +223 255 1 +231 255 1 +239 255 1 +247 255 1 +255 255 1 +255 249 1 +255 244 1 +255 239 1 +255 233 1 +255 228 1 +255 223 1 +255 217 1 +255 212 1 +255 207 1 +255 201 1 +255 196 1 +255 191 1 +255 185 1 +255 180 1 +255 175 1 +255 170 1 +255 164 1 +255 159 1 +255 154 1 +255 148 1 +255 143 1 +255 138 1 +255 132 1 +255 127 1 +255 122 1 +255 116 1 +255 111 1 +255 106 1 +255 100 1 +255 95 1 +255 90 1 +255 85 1 +255 79 1 +255 74 1 +255 69 1 +255 63 1 +255 58 1 +255 53 1 +255 47 1 +255 42 1 +255 37 1 +255 31 1 +255 26 1 +255 21 1 +255 15 1 +255 10 1 +255 5 1 +255 1 1 +255 1 15 +255 1 31 +255 1 47 +255 1 63 +255 1 79 +255 1 95 +255 1 111 +255 1 127 +255 1 143 +255 1 159 +255 1 175 +255 1 191 +255 1 207 +255 1 223 +255 1 239 From 2e30fe574ca435f9e19c4034d9b852b3b56de580 Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 5 Jan 2016 15:00:37 -0500 Subject: [PATCH 48/50] update authors list --- AUTHORS.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index 9f2131d..20bd3c7 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -5,7 +5,5 @@ Scisco https://github.com/scisco Marc Farra https://github.com/kamicut Drew Bollinger https://github.com/drewbo Alex Kappel https://twitter.com/alex_kappel -Eric Miller https://github.com/millerEric -Yuriy Czoli https://github.com/YKCzoli See also https://github.com/developmentseed/landsat-util/graphs/contributors. From 7035cc2973e8b0fd0fc57ccbb0829fdeaad20a25 Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 5 Jan 2016 15:00:52 -0500 Subject: [PATCH 49/50] bump up to version 0.10.0 --- CHANGES.txt | 13 +++++++++++++ landsat/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index e5aadcc..3243704 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,19 @@ Changes ======= +0.10.0 (2016-01-05) +------------------ +- add support for bare json output +- faster travis tests +- add street search +- apply ndvigrey to process and download +- fixes #127 +- update commands help file +- apply pansharpen and ndvi only if -p is used +- download zip if bands are not specified +- better handle url joins to a fix error on Windows +- other small bug fixes + 0.9.1 (2015-10-26) ------------------ - Add missing package (polyline) to setup.py diff --git a/landsat/__init__.py b/landsat/__init__.py index ca16de9..9d1bb72 100644 --- a/landsat/__init__.py +++ b/landsat/__init__.py @@ -1 +1 @@ -__version__ = '0.10b3' +__version__ = '0.10.0' From 1175020289452fc8036b15f4f40c1b0de155c7ed Mon Sep 17 00:00:00 2001 From: Scisco Date: Tue, 5 Jan 2016 15:27:19 -0500 Subject: [PATCH 50/50] include color maps in manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 5f54a81..23d7d34 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,4 +7,4 @@ recursive-exclude * __pycache__ recursive-exclude * *.py[co] recursive-include docs *.rst conf.py Makefile make.bat -recursive-include landsat/tests/samples *.tar.bz2 +recursive-include landsat/maps *.txt