Skip to content

Commit 016bb3e

Browse files
begoldsmbegoldsm
authored
Merge pull request #144 from begoldsm/master
Update to Version 0.0.4
2 parents c85e75f + 6e6540d commit 016bb3e

File tree

76 files changed

+33733
-8
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+33733
-8
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,5 @@ build/
77
dist/
88
tests/__pycache__/
99
*.suo
10-
publish/
10+
publish/
11+
adlsenv/

‎HISTORY.rst‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,17 @@
22
33
Release History
44
===============
5+
0.0.4 (2017-02-07)
6+
------------------
7+
* Fix for folder upload to properly delete folders with contents when overwrite specified.
8+
* Fix to set verbose output to False/Off by default. This removes progress tracking output by default but drastically improves performance.
59

6-
0.0.2 (2017-01-30)
10+
0.0.3 (2017-02-02)
711
------------------
12+
* Fix to setup.py to include the HISTORY.rst file. No other changes
813

14+
0.0.2 (2017-01-30)
15+
------------------
916
* Addresses an issue with lib.auth() not properly defaulting to 2FA
1017
* Fixes an issue with Overwrite for ADLUploader sometimes not being honored.
1118
* Fixes an issue with empty files not properly being uploaded and resulting in a hang in progress tracking.

‎azure/datalake/store/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
# license information.
77
# --------------------------------------------------------------------------
88

9-
__version__ = "0.0.3"
9+
__version__ = "0.0.4"
1010

1111
from .core import AzureDLFileSystem
1212
from .multithread import ADLDownloader

‎azure/datalake/store/multithread.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ class ADLDownloader(object):
100100
"""
101101
def __init__(self, adlfs, rpath, lpath, nthreads=None, chunksize=2**28,
102102
buffersize=2**22, blocksize=2**22, client=None, run=True,
103-
overwrite=False, verbose=True):
103+
overwrite=False, verbose=False):
104104
if not overwrite and os.path.exists(lpath):
105105
raise FileExistsError(lpath)
106106
if client:
@@ -315,14 +315,14 @@ class ADLUploader(object):
315315
"""
316316
def __init__(self, adlfs, rpath, lpath, nthreads=None, chunksize=2**28,
317317
buffersize=2**22, blocksize=2**22, client=None, run=True,
318-
overwrite=False, verbose=True):
318+
overwrite=False, verbose=False):
319319
if not overwrite and adlfs.exists(rpath):
320320
raise FileExistsError(rpath)
321321

322322
# forcibly remove the target file before execution
323323
# if the user indicates they want to overwrite the destination.
324324
if overwrite and adlfs.exists(rpath):
325-
adlfs.remove(rpath)
325+
adlfs.remove(rpath, True)
326326

327327
if client:
328328
self.client = client
@@ -490,7 +490,7 @@ def merge_chunks(adlfs, outfile, files, shutdown_event=None, overwrite=False):
490490
# and concat, we will remove it if the user specified overwrite.
491491
if adlfs.exists(outfile):
492492
if overwrite:
493-
adlfs.remove(outfile)
493+
adlfs.remove(outfile, True)
494494
else:
495495
raise FileExistsError(outfile)
496496

‎azure/datalake/store/transfer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ class ADLTransferClient(object):
227227
def __init__(self, adlfs, transfer, merge=None, nthreads=None,
228228
chunksize=2**28, blocksize=2**25, chunked=True,
229229
unique_temporary=True, delimiter=None,
230-
parent=None, verbose=True, buffersize=2**25):
230+
parent=None, verbose=False, buffersize=2**25):
231231
self._adlfs = adlfs
232232
self._parent = parent
233233
self._transfer = transfer

‎tests/recordings/test_cli/test_cat.yaml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,4 +664,137 @@ interactions:
664664
x-ms-request-id: [7f61ac06-caa9-46cd-a21e-80e1cbe79a45]
665665
x-ms-webhdfs-version: [16.07.18.01]
666666
status: {code: 200, message: OK}
667+
- request:
668+
body: '123456'
669+
headers:
670+
Accept: ['*/*']
671+
Accept-Encoding: ['gzip, deflate']
672+
Connection: [keep-alive]
673+
Content-Length: ['6']
674+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
675+
Azure-Data-Lake-Store-SDK-For-Python]
676+
x-ms-client-request-id: [3acf3866-ea4e-11e6-ba8e-645106422854]
677+
method: PUT
678+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true
679+
response:
680+
body: {string: ''}
681+
headers:
682+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
683+
Content-Length: ['0']
684+
ContentLength: ['0']
685+
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
686+
Expires: ['-1']
687+
Location: ['https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true']
688+
Pragma: [no-cache]
689+
Status: ['0x0']
690+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
691+
X-Content-Type-Options: [nosniff]
692+
x-ms-request-id: [9148b9e7-33c8-4910-bc1a-b0b1f842cbae]
693+
x-ms-webhdfs-version: [16.07.18.01]
694+
status: {code: 201, message: Created}
695+
- request:
696+
body: null
697+
headers:
698+
Accept: ['*/*']
699+
Accept-Encoding: ['gzip, deflate']
700+
Connection: [keep-alive]
701+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
702+
Azure-Data-Lake-Store-SDK-For-Python]
703+
x-ms-client-request-id: [3b2deb14-ea4e-11e6-82ba-645106422854]
704+
method: GET
705+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo?OP=LISTSTATUS
706+
response:
707+
body: {string: '{"FileStatuses":{"FileStatus":[{"length":6,"pathSuffix":"bar","type":"FILE","blockSize":268435456,"accessTime":1486153240013,"modificationTime":1486153240091,"replication":1,"permission":"770","owner":"9a23860e-03b0-4bad-a8b7-e1d081d592bd","group":"2e6c02d2-a364-4530-9137-d17403996cbf"}]}}'}
708+
headers:
709+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
710+
Content-Length: ['290']
711+
Content-Type: [application/json; charset=utf-8]
712+
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
713+
Expires: ['-1']
714+
Pragma: [no-cache]
715+
Status: ['0x0']
716+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
717+
X-Content-Type-Options: [nosniff]
718+
x-ms-request-id: [6714f3f1-dbee-4f18-a291-532e63624201]
719+
x-ms-webhdfs-version: [16.07.18.01]
720+
status: {code: 200, message: OK}
721+
- request:
722+
body: null
723+
headers:
724+
Accept: ['*/*']
725+
Accept-Encoding: ['gzip, deflate']
726+
Connection: [keep-alive]
727+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
728+
Azure-Data-Lake-Store-SDK-For-Python]
729+
x-ms-client-request-id: [3b3fc49c-ea4e-11e6-9ac6-645106422854]
730+
method: GET
731+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?read=true&OP=OPEN&length=6&offset=0
732+
response:
733+
body: {string: '123456'}
734+
headers:
735+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
736+
Content-Type: [application/octet-stream]
737+
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
738+
Expires: ['-1']
739+
Pragma: [no-cache]
740+
Status: ['0x0']
741+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
742+
Transfer-Encoding: [chunked]
743+
X-Content-Type-Options: [nosniff]
744+
x-ms-request-id: [94355725-6a34-4b69-9a6a-139a47e404b3]
745+
x-ms-webhdfs-version: [16.07.18.01]
746+
status: {code: 200, message: OK}
747+
- request:
748+
body: null
749+
headers:
750+
Accept: ['*/*']
751+
Accept-Encoding: ['gzip, deflate']
752+
Connection: [keep-alive]
753+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
754+
Azure-Data-Lake-Store-SDK-For-Python]
755+
x-ms-client-request-id: [3b7839cc-ea4e-11e6-98ad-645106422854]
756+
method: GET
757+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo?OP=LISTSTATUS
758+
response:
759+
body: {string: '{"FileStatuses":{"FileStatus":[{"length":6,"pathSuffix":"bar","type":"FILE","blockSize":268435456,"accessTime":1486153240013,"modificationTime":1486153240091,"replication":1,"permission":"770","owner":"9a23860e-03b0-4bad-a8b7-e1d081d592bd","group":"2e6c02d2-a364-4530-9137-d17403996cbf"}]}}'}
760+
headers:
761+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
762+
Content-Length: ['290']
763+
Content-Type: [application/json; charset=utf-8]
764+
Date: ['Fri, 03 Feb 2017 20:20:39 GMT']
765+
Expires: ['-1']
766+
Pragma: [no-cache]
767+
Status: ['0x0']
768+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
769+
X-Content-Type-Options: [nosniff]
770+
x-ms-request-id: [5e91eec5-fcda-4670-88ee-22b73a97fcda]
771+
x-ms-webhdfs-version: [16.07.18.01]
772+
status: {code: 200, message: OK}
773+
- request:
774+
body: null
775+
headers:
776+
Accept: ['*/*']
777+
Accept-Encoding: ['gzip, deflate']
778+
Connection: [keep-alive]
779+
Content-Length: ['0']
780+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
781+
Azure-Data-Lake-Store-SDK-For-Python]
782+
x-ms-client-request-id: [3b8b2542-ea4e-11e6-b647-645106422854]
783+
method: DELETE
784+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=DELETE&recursive=False
785+
response:
786+
body: {string: '{"boolean":true}'}
787+
headers:
788+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
789+
Content-Length: ['16']
790+
Content-Type: [application/json; charset=utf-8]
791+
Date: ['Fri, 03 Feb 2017 20:20:40 GMT']
792+
Expires: ['-1']
793+
Pragma: [no-cache]
794+
Status: ['0x0']
795+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
796+
X-Content-Type-Options: [nosniff]
797+
x-ms-request-id: [43c19409-7c02-49da-b4e3-820df01cd77a]
798+
x-ms-webhdfs-version: [16.07.18.01]
799+
status: {code: 200, message: OK}
667800
version: 1

‎tests/recordings/test_cli/test_chgrp.yaml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,4 +562,111 @@ interactions:
562562
x-ms-request-id: [e55b56ae-05e1-46e1-a8cc-6df30929644a]
563563
x-ms-webhdfs-version: [16.07.18.01]
564564
status: {code: 200, message: OK}
565+
- request:
566+
body: '123456'
567+
headers:
568+
Accept: ['*/*']
569+
Accept-Encoding: ['gzip, deflate']
570+
Connection: [keep-alive]
571+
Content-Length: ['6']
572+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
573+
Azure-Data-Lake-Store-SDK-For-Python]
574+
x-ms-client-request-id: [3bb17210-ea4e-11e6-8096-645106422854]
575+
method: PUT
576+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true
577+
response:
578+
body: {string: ''}
579+
headers:
580+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
581+
Content-Length: ['0']
582+
ContentLength: ['0']
583+
Date: ['Fri, 03 Feb 2017 20:20:41 GMT']
584+
Expires: ['-1']
585+
Location: ['https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=CREATE&write=true&overwrite=true']
586+
Pragma: [no-cache]
587+
Status: ['0x0']
588+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
589+
X-Content-Type-Options: [nosniff]
590+
x-ms-request-id: [9f0a0b4b-0f61-454b-8885-1382978a24f6]
591+
x-ms-webhdfs-version: [16.07.18.01]
592+
status: {code: 201, message: Created}
593+
- request:
594+
body: null
595+
headers:
596+
Accept: ['*/*']
597+
Accept-Encoding: ['gzip, deflate']
598+
Connection: [keep-alive]
599+
Content-Length: ['0']
600+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
601+
Azure-Data-Lake-Store-SDK-For-Python]
602+
x-ms-client-request-id: [3c25be28-ea4e-11e6-9eac-645106422854]
603+
method: PUT
604+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=SETOWNER&group=foo
605+
response:
606+
body: {string: ''}
607+
headers:
608+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
609+
Content-Length: ['0']
610+
Date: ['Fri, 03 Feb 2017 20:20:41 GMT']
611+
Expires: ['-1']
612+
Pragma: [no-cache]
613+
Status: ['0x0']
614+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
615+
X-Content-Type-Options: [nosniff]
616+
x-ms-request-id: [747d9d1d-22ce-48f3-a981-ad656dedf8ce]
617+
x-ms-webhdfs-version: [16.07.18.01]
618+
status: {code: 200, message: OK}
619+
- request:
620+
body: null
621+
headers:
622+
Accept: ['*/*']
623+
Accept-Encoding: ['gzip, deflate']
624+
Connection: [keep-alive]
625+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
626+
Azure-Data-Lake-Store-SDK-For-Python]
627+
x-ms-client-request-id: [3c354ea4-ea4e-11e6-8970-645106422854]
628+
method: GET
629+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo?OP=LISTSTATUS
630+
response:
631+
body: {string: '{"FileStatuses":{"FileStatus":[{"length":6,"pathSuffix":"bar","type":"FILE","blockSize":268435456,"accessTime":1486153241462,"modificationTime":1486153241538,"replication":1,"permission":"770","owner":"9a23860e-03b0-4bad-a8b7-e1d081d592bd","group":"2e6c02d2-a364-4530-9137-d17403996cbf"}]}}'}
632+
headers:
633+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
634+
Content-Length: ['290']
635+
Content-Type: [application/json; charset=utf-8]
636+
Date: ['Fri, 03 Feb 2017 20:20:41 GMT']
637+
Expires: ['-1']
638+
Pragma: [no-cache]
639+
Status: ['0x0']
640+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
641+
X-Content-Type-Options: [nosniff]
642+
x-ms-request-id: [a5680751-42c8-462f-ac41-704ac6f26bf9]
643+
x-ms-webhdfs-version: [16.07.18.01]
644+
status: {code: 200, message: OK}
645+
- request:
646+
body: null
647+
headers:
648+
Accept: ['*/*']
649+
Accept-Encoding: ['gzip, deflate']
650+
Connection: [keep-alive]
651+
Content-Length: ['0']
652+
User-Agent: [python/3.5.2 (Windows-10-10.0.14393-SP0) azure.datalake.store.lib/0.0.4
653+
Azure-Data-Lake-Store-SDK-For-Python]
654+
x-ms-client-request-id: [3c468cca-ea4e-11e6-9030-645106422854]
655+
method: DELETE
656+
uri: https://fakestore.azuredatalakestore.net/webhdfs/v1/azure_test_dir/foo/bar?OP=DELETE&recursive=False
657+
response:
658+
body: {string: '{"boolean":true}'}
659+
headers:
660+
Cache-Control: ['no-cache, no-cache, no-store, max-age=0']
661+
Content-Length: ['16']
662+
Content-Type: [application/json; charset=utf-8]
663+
Date: ['Fri, 03 Feb 2017 20:20:42 GMT']
664+
Expires: ['-1']
665+
Pragma: [no-cache]
666+
Status: ['0x0']
667+
Strict-Transport-Security: [max-age=15724800; includeSubDomains]
668+
X-Content-Type-Options: [nosniff]
669+
x-ms-request-id: [9e2c78c7-10ad-482e-9794-ccfa4e3454a9]
670+
x-ms-webhdfs-version: [16.07.18.01]
671+
status: {code: 200, message: OK}
565672
version: 1

0 commit comments

Comments
 (0)