Skip to content

Commit

Permalink
Merge pull request #173 from developmentseed/sgillies-py3
Browse files Browse the repository at this point in the history
Python 3 is now supported
  • Loading branch information
Scisco committed Mar 25, 2016
2 parents 2ad1aa7 + d844195 commit 5af669b
Show file tree
Hide file tree
Showing 22 changed files with 204 additions and 106 deletions.
54 changes: 54 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
language: python
sudo: false

python:
- '2.7'
- '3.5'

cache:
directories:
- ~/.cache/pip

addons:
apt:
packages:
- libgdal1h
- gdal-bin
- libproj-dev
- libhdf5-serial-dev
- libpng-dev
- libgdal-dev
- libatlas-dev
- libatlas-base-dev
- gfortran

env:
global:
- secure: QsF7ignSAbH/WCyO6v9bw1exmCWDQR0DqmHkwJ5swc9N44OOOzbWGsaMSYB5y9h+d70fz4arbxQDhsk2KvX4Zd1/2YIMOrIsbgDYeegpkhVPgyQNPKmVqiX+Tb47t1C/TgkC7A07tiPpuefYcLNMZ8gzz7oKhh1UKapYftqzZ+g=
- secure: HxjeKWSROBQYy9NuNkgQeaK1ubTF8vH5FcR8nUTSAYxxw/qOzKpqkiq4BcJSRcIwTbkvaBf4MshLGVOxPjMeyJFe06UD/6LvTUGS3bwdya+m0RFjHe5/3wzS8/MxLbTlvgzmuGLLKOsJjXCi9eQQchKfHv+QuhGxhYVLQpnbU9E=
- secure: Zq0Z2UA2A7/ieXX8XoMweClJTp8hiVBxoQ1ylJYNd7qsRSk0QvZhn62db5/x48L9S1kELk0sG64q5Pf96/RPLpdjkBUAdEkS7qF+QOvRvAv2woNEHutjlMUvP6jwYGbug+AORg76btZ57OwMOi3aTkagQMMKnokfo7KGbffy0Jo=
- PIP_WHEEL_DIR=$HOME/.cache/pip/wheels
- PIP_FIND_LINKS=file://$HOME/.cache/pip/wheels

before_install:
- pip install -U pip
- pip install wheel


install:
- pip install -r requirements-dev.txt
- pip install -e .


script:
- python setup.py test

deploy:
provider: pypi
user: devseed
password:
secure: WtawFW/999XYszmZfj1Qk82l00OSyP2JBVOOGCERrW1gVO7MYtYsgP31HKRSzNTCTHJNVDpdK4WZWY6zPQqC3l2UfWYYsvRn0hCoI8AJxE5VCUEg6Ccpe6fMJuhp1pq6Zy7yrfBSZcOB9aqSHLBJsunD2o3mNlTC8WV8vNK74ck=
on:
repo: developmentseed/landsat-util
branch:
- master
1 change: 1 addition & 0 deletions AUTHORS.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ Authors
Scisco https://github.com/scisco
Marc Farra https://github.com/kamicut
Drew Bollinger https://github.com/drewbo
Sean Gillies https://github.com/sgillies
Alex Kappel https://twitter.com/alex_kappel

See also https://github.com/developmentseed/landsat-util/graphs/contributors.
6 changes: 5 additions & 1 deletion CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
Changes
=======

0.12.2 (2016-02-18)
0.13.0 (2016-03-25)
------------------
- Python 3.5 support

0.12.2 (2016-03-24)
------------------
- Fix for #167
- Fix for #145
Expand Down
2 changes: 2 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
include AUTHORS.txt
include LICENSE
include README.md
include requirements.txt
include requirements-dev.txt

recursive-include tests *
recursive-exclude * __pycache__
Expand Down
4 changes: 2 additions & 2 deletions README.rst
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
Landsat-util
===============

.. image:: https://circleci.com/gh/developmentseed/landsat-util/tree/develop.svg?style=svg
:target: https://circleci.com/gh/developmentseed/landsat-util/tree/develop
.. image:: https://travis-ci.org/developmentseed/landsat-util.svg?branch=master
:target: https://travis-ci.org/developmentseed/landsat-util

.. image:: https://badge.fury.io/py/landsat-util.svg
:target: http://badge.fury.io/py/landsat-util
Expand Down
29 changes: 0 additions & 29 deletions circle.yml

This file was deleted.

2 changes: 1 addition & 1 deletion landsat/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.12.2'
__version__ = '0.13.0'
1 change: 1 addition & 0 deletions landsat/decorators.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import warnings

import rasterio


Expand Down
9 changes: 6 additions & 3 deletions landsat/downloader.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
# Landsat Util
# License: CC0 1.0 Universal

from __future__ import print_function, division, absolute_import

from xml.etree import ElementTree
from os.path import join, exists, getsize

import requests
from usgs import api, USGSError
from homura import download as fetch

from utils import check_create_folder, url_builder
from mixins import VerbosityMixin
import settings
from .utils import check_create_folder, url_builder
from .mixins import VerbosityMixin
from . import settings


class RemoteFileDoesntExist(Exception):
Expand Down
8 changes: 5 additions & 3 deletions landsat/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# Landsat Util
# License: CC0 1.0 Universal

from __future__ import print_function, division, absolute_import

import os
import tarfile
import glob
Expand All @@ -20,9 +22,9 @@
from skimage.exposure import rescale_intensity
from polyline.codec import PolylineCodec

from mixins import VerbosityMixin
from utils import get_file, check_create_folder, exit, adjust_bounding_box
from decorators import rasterio_decorator
from .mixins import VerbosityMixin
from .utils import get_file, check_create_folder, exit, adjust_bounding_box
from .decorators import rasterio_decorator


class FileDoesNotExist(Exception):
Expand Down
34 changes: 20 additions & 14 deletions landsat/landsat.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,33 @@
# Landsat Util
# License: CC0 1.0 Universal

from __future__ import print_function, division, absolute_import

import argparse
import textwrap
import json
from os.path import join
from urllib2 import URLError

try:
from urllib.request import URLError
except ImportError:
from urllib2 import URLError

from datetime import datetime
from dateutil.relativedelta import relativedelta
from dateutil.parser import parse
import pycurl
from boto.exception import NoAuthHandlerFound

from downloader import Downloader, IncorrectSceneId, RemoteFileDoesntExist, USGSInventoryAccessMissing
from search import Search
from uploader import Uploader
from utils import reformat_date, convert_to_integer_list, timer, exit, get_file, convert_to_float_list
from mixins import VerbosityMixin
from image import Simple, PanSharpen, FileDoesNotExist
from ndvi import NDVIWithManualColorMap, NDVI
from __init__ import __version__
import settings
from .downloader import Downloader, IncorrectSceneId, RemoteFileDoesntExist, USGSInventoryAccessMissing
from .search import Search
from .uploader import Uploader
from .utils import reformat_date, convert_to_integer_list, timer, exit, get_file, convert_to_float_list
from .mixins import VerbosityMixin
from .image import Simple, PanSharpen, FileDoesNotExist
from .ndvi import NDVIWithManualColorMap, NDVI
from .__init__ import __version__
from . import settings


DESCRIPTION = """Landsat-util is a command line utility that makes it easy to
Expand Down Expand Up @@ -465,10 +471,10 @@ def process_image(path, bands=None, verbose=False, pansharpen=False, ndvi=False,
p = Simple(path, bands=bands, dst_path=settings.PROCESSED_IMAGE, verbose=verbose, force_unzip=force_unzip,
bounds=bounds)

except IOError as e:
exit(e.message, 1)
except FileDoesNotExist as e:
exit(e.message, 1)
except IOError as err:
exit(str(err), 1)
except FileDoesNotExist as err:
exit(str(err), 1)

return p.run()

Expand Down
4 changes: 3 additions & 1 deletion landsat/mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# Landsat Util
# License: CC0 1.0 Universal

from __future__ import print_function, division, absolute_import

import sys
import subprocess
from termcolor import colored
Expand Down Expand Up @@ -108,6 +110,6 @@ def _print(self, msg, color=None, arrow=False, indent=None):
if indent:
msg = (' ' * indent) + msg

sys.stdout.write(msg + '\n')
print(msg)

return msg
10 changes: 6 additions & 4 deletions landsat/ndvi.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
from __future__ import print_function, division, absolute_import

from os.path import join

import rasterio
import numpy

import settings
from decorators import rasterio_decorator
from image import BaseProcess
from . import settings
from .decorators import rasterio_decorator
from .image import BaseProcess


class NDVI(BaseProcess):
Expand Down Expand Up @@ -45,7 +47,7 @@ def _read_cmap(self):
except IOError:
pass

self.cmap = {k: v[:4] for k, v in colormap.iteritems()}
self.cmap = {k: v[:4] for k, v in colormap.items()}

@rasterio_decorator
def run(self):
Expand Down
6 changes: 4 additions & 2 deletions landsat/search.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
# Landsat Util
# License: CC0 1.0 Universal

from __future__ import print_function, division, absolute_import

import json
import time
import requests

import settings
from utils import three_digit, create_paired_list, geocode
from . import settings
from .utils import three_digit, create_paired_list, geocode


class Search(object):
Expand Down
33 changes: 21 additions & 12 deletions landsat/uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,32 @@

# The S3 uploader is a fork of pys3upload (https://github.com/leetreveil/pys3upload)

from __future__ import print_function, division
from __future__ import print_function, division, absolute_import

import os
import sys
import time
import threading
import contextlib
import Queue

try:
import queue
except:
import Queue as queue

from multiprocessing import pool

try:
import cStringIO
StringIO = cStringIO
from io import BytesIO as StringIO
except ImportError:
import StringIO
try:
from cStringIO import StringIO
except:
from StringIO import StringIO

from boto.s3.connection import S3Connection
from mixins import VerbosityMixin

from .mixins import VerbosityMixin

STREAM = sys.stderr

Expand Down Expand Up @@ -96,7 +105,7 @@ def cb(part_no, uploaded, total):

self.output('Uploading to S3', normal=True, arrow=True)
upload(bucket_name, self.key, self.secret,
data_collector(f.readlines()), filename, cb,
data_collector(iter(f)), filename, cb,
threads=10, replace=True, secure=True, connection=self.conn)

print('\n')
Expand All @@ -114,7 +123,7 @@ def data_collector(iterable, def_buf_size=5242880):
:returns:
A generator object
"""
buf = ''
buf = b''
for data in iterable:
buf += data
if len(buf) >= def_buf_size:
Expand All @@ -130,11 +139,11 @@ def upload_part(upload_func, progress_cb, part_no, part_data):

def _upload_part(retries_left=num_retries):
try:
with contextlib.closing(StringIO.StringIO(part_data)) as f:
with contextlib.closing(StringIO(part_data)) as f:
f.seek(0)
cb = lambda c, t: progress_cb(part_no, c, t) if progress_cb else None
upload_func(f, part_no, cb=cb, num_cb=100)
except Exception, exc:
except Exception as exc:
retries_left -= 1
if retries_left > 0:
return _upload_part(retries_left=retries_left)
Expand Down Expand Up @@ -208,7 +217,7 @@ def upload(bucket, aws_access_key, aws_secret_key,
raise Exception('s3 key ' + key + ' already exists')

multipart_obj = b.initiate_multipart_upload(key)
err_queue = Queue.Queue()
err_queue = queue.Queue()
lock = threading.Lock()
upload.counter = 0

Expand All @@ -218,7 +227,7 @@ def upload(bucket, aws_access_key, aws_secret_key,
def check_errors():
try:
exc = err_queue.get(block=False)
except Queue.Empty:
except queue.Empty:
pass
else:
raise exc
Expand Down
Loading

0 comments on commit 5af669b

Please sign in to comment.