Skip to content

Commit

Permalink
Update to version 0.0.4
Browse files Browse the repository at this point in the history 
Fixes:
* Overwrite functioning if the destination is a folder with contents
* Fixes to remove verbose output by default, which improves upload and
download performance
  • Loading branch information
begoldsm committed Feb 7, 2017
1 parent c85e75f commit 7427095
Show file tree
Hide file tree
Showing 75 changed files with 33,725 additions and 7 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ build/
dist/
tests/__pycache__/
*.suo
publish/
publish/
adlsenv/
2 changes: 1 addition & 1 deletion azure/datalake/store/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# license information.
# --------------------------------------------------------------------------

__version__ = "0.0.3"
__version__ = "0.0.4"

from .core import AzureDLFileSystem
from .multithread import ADLDownloader
Expand Down
8 changes: 4 additions & 4 deletions azure/datalake/store/multithread.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ class ADLDownloader(object):
"""
def __init__(self, adlfs, rpath, lpath, nthreads=None, chunksize=2**28,
buffersize=2**22, blocksize=2**22, client=None, run=True,
overwrite=False, verbose=True):
overwrite=False, verbose=False):
if not overwrite and os.path.exists(lpath):
raise FileExistsError(lpath)
if client:
Expand Down Expand Up @@ -315,14 +315,14 @@ class ADLUploader(object):
"""
def __init__(self, adlfs, rpath, lpath, nthreads=None, chunksize=2**28,
buffersize=2**22, blocksize=2**22, client=None, run=True,
overwrite=False, verbose=True):
overwrite=False, verbose=False):
if not overwrite and adlfs.exists(rpath):
raise FileExistsError(rpath)

# forcibly remove the target file before execution
# if the user indicates they want to overwrite the destination.
if overwrite and adlfs.exists(rpath):
adlfs.remove(rpath)
adlfs.remove(rpath, True)

if client:
self.client = client
Expand Down Expand Up @@ -490,7 +490,7 @@ def merge_chunks(adlfs, outfile, files, shutdown_event=None, overwrite=False):
# and concat, we will remove it if the user specified overwrite.
if adlfs.exists(outfile):
if overwrite:
adlfs.remove(outfile)
adlfs.remove(outfile, True)
else:
raise FileExistsError(outfile)

Expand Down
2 changes: 1 addition & 1 deletion azure/datalake/store/transfer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ class ADLTransferClient(object):
def __init__(self, adlfs, transfer, merge=None, nthreads=None,
chunksize=2**28, blocksize=2**25, chunked=True,
unique_temporary=True, delimiter=None,
parent=None, verbose=True, buffersize=2**25):
parent=None, verbose=False, buffersize=2**25):
self._adlfs = adlfs
self._parent = parent
self._transfer = transfer
Expand Down
133 changes: 133 additions & 0 deletions tests/recordings/test_cli/test_cat.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,4 +664,137 @@ interactions:
x-ms-request-id: [7f61ac06-caa9-46cd-a21e-80e1cbe79a45]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: '123456'
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['6']
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3acf3866-ea4e-11e6-ba8e-645106422854]
method: PUT
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true
response:
body: {string: ''}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['0']
ContentLength: ['0']
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
Expires: ['-1']
Location: ['https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [9148b9e7-33c8-4910-bc1a-b0b1f842cbae]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 201, message: Created}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3b2deb14-ea4e-11e6-82ba-645106422854]
method: GET
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo?OP=LISTSTATUS
response:
body: {string: '{"FileStatuses":{"FileStatus":[{"length":6,"pathSuffix":"bar","type":"FILE","blockSize":268435456,"accessTime":1486153240013,"modificationTime":1486153240091,"replication":1,"permission":"770","owner":"9a23860e-03b0-4bad-a8b7-e1d081d592bd","group":"2e6c02d2-a364-4530-9137-d17403996cbf"}]}}'}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['290']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [6714f3f1-dbee-4f18-a291-532e63624201]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3b3fc49c-ea4e-11e6-9ac6-645106422854]
method: GET
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?read=true&OP=OPEN&length=6&offset=0
response:
body: {string: '123456'}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Type: [application/octet-stream]
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
Transfer-Encoding: [chunked]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [94355725-6a34-4b69-9a6a-139a47e404b3]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3b7839cc-ea4e-11e6-98ad-645106422854]
method: GET
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo?OP=LISTSTATUS
response:
body: {string: '{"FileStatuses":{"FileStatus":[{"length":6,"pathSuffix":"bar","type":"FILE","blockSize":268435456,"accessTime":1486153240013,"modificationTime":1486153240091,"replication":1,"permission":"770","owner":"9a23860e-03b0-4bad-a8b7-e1d081d592bd","group":"2e6c02d2-a364-4530-9137-d17403996cbf"}]}}'}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['290']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [5e91eec5-fcda-4670-88ee-22b73a97fcda]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['0']
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3b8b2542-ea4e-11e6-b647-645106422854]
method: DELETE
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=DELETE&recursive=False
response:
body: {string: '{"boolean":true}'}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['16']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 03 Feb 2017 20:20:40 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [43c19409-7c02-49da-b4e3-820df01cd77a]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
version: 1
107 changes: 107 additions & 0 deletions tests/recordings/test_cli/test_chgrp.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,4 +562,111 @@ interactions:
x-ms-request-id: [e55b56ae-05e1-46e1-a8cc-6df30929644a]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: '123456'
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['6']
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3bb17210-ea4e-11e6-8096-645106422854]
method: PUT
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true
response:
body: {string: ''}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['0']
ContentLength: ['0']
Date: ['Fri, 03 Feb 2017 20:20:41 GMT']
Expires: ['-1']
Location: ['https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [9f0a0b4b-0f61-454b-8885-1382978a24f6]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 201, message: Created}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['0']
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3c25be28-ea4e-11e6-9eac-645106422854]
method: PUT
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=SETOWNER&group=foo
response:
body: {string: ''}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['0']
Date: ['Fri, 03 Feb 2017 20:20:41 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [747d9d1d-22ce-48f3-a981-ad656dedf8ce]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3c354ea4-ea4e-11e6-8970-645106422854]
method: GET
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo?OP=LISTSTATUS
response:
body: {string: '{"FileStatuses":{"FileStatus":[{"length":6,"pathSuffix":"bar","type":"FILE","blockSize":268435456,"accessTime":1486153241462,"modificationTime":1486153241538,"replication":1,"permission":"770","owner":"9a23860e-03b0-4bad-a8b7-e1d081d592bd","group":"2e6c02d2-a364-4530-9137-d17403996cbf"}]}}'}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['290']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 03 Feb 2017 20:20:41 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [a5680751-42c8-462f-ac41-704ac6f26bf9]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
- request:
body: null
headers:
Accept: ['*/*']
Accept-Encoding: ['gzip, deflate']
Connection: [keep-alive]
Content-Length: ['0']
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
Azure-Data-Lake-Store-SDK-For-Python]
x-ms-client-request-id: [3c468cca-ea4e-11e6-9030-645106422854]
method: DELETE
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=DELETE&recursive=False
response:
body: {string: '{"boolean":true}'}
headers:
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
Content-Length: ['16']
Content-Type: [application/json; charset=utf-8]
Date: ['Fri, 03 Feb 2017 20:20:42 GMT']
Expires: ['-1']
Pragma: [no-cache]
Status: ['0x0']
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
X-Content-Type-Options: [nosniff]
x-ms-request-id: [9e2c78c7-10ad-482e-9794-ccfa4e3454a9]
x-ms-webhdfs-version: [16.07.18.01]
status: {code: 200, message: OK}
version: 1
Loading

0 comments on commit 7427095

Please sign in to comment.