Merge pull request #1 from apache/trunk

Synicng

Author: Eis-D-Z

.travis.yml

@@ -1,6 +1,5 @@
 language: python
-sudo: false
-dist: trusty
+dist: xenial
 os: linux
 
 # Note: graphviz is needed for grapgs embedded in the docs
@@ -13,41 +12,52 @@ addons:
 matrix:
   fast_finish: true
   include:
-    - python: 2.7
-      before_script: TOX_ENV=py2.7,py2.7-dist,py2.7-dist-wheel
-    - python: 3.4
-    - python: 3.5
-    - python: 3.6
-    - python: 3.7
-      dist: xenial
-      sudo: required
+    - name: Unit Tests (Python 3.5)
+      python: 3.5
+    - name: Unit Tests (Python 3.6)
+      python: 3.6
+    - name: Unit Tests, Installation Checks (Python 3.7)
+      python: 3.7
       before_script: TOX_ENV=py3.7,py3.7-dist,py3.7-dist-wheel
-    - python: 3.8
-      dist: xenial
-      sudo: required
-    - python: pypy
-    - python: pypy3
-    - env: ENV=checks
-      python: 2.7
+    - name: Unit Tests (Python 3.8 on Windows)
+      os: windows
+      python: 3.8
+      language: shell
+      before_install:
+        - choco install python --version 3.8.1
+        - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+        - python -m pip install --upgrade pip
+      before_script: TOX_ENV=py3.8-windows
+      env: PATH=/c/Python38:/c/Python38/Scripts:$PATH
+    - name: Unit Tests (Python 3.8)
+      python: 3.8
+    - name: Unit Tests (PyPy 3.5)
+      python: pypy3.5
+    - name: License and File name Checks
+      env: ENV=checks
+      python: 3.7
       before_script: TOX_ENV=checks
-    - env: ENV=lint
-      python: 2.7
+    - name: Lint (flake8)
+      env: ENV=lint
+      python: 3.7
       before_script: TOX_ENV=lint
-    - env: ENV=pylint
-      python: 2.7
+    - name: Lint (pylint)
+      env: ENV=pylint
+      python: 3.7
       before_script: TOX_ENV=pylint
-    - env: ENV=mypy
+    - name: MyPy
+      env: ENV=mypy
       python: 3.7
-      dist: xenial
-      sudo: required
       before_script: TOX_ENV=mypy
-    - env: ENV=coverage
-      python: 2.7
+    - name: Code Coverage (Unit tests)
+      env: ENV=coverage
+      python: 3.7
       before_script: TOX_ENV=coverage-travis
-    - env:
+    - name: Documentation
+      env:
         - ENV=docs
         - secure: "d4F2RE2iWFh6yqE5MUxnakDihBMsY/hMsaMWllVAYU/fvCMKhaW97d51EiS55kur/eNVX2jpEqkVkSu5VeNSxC3w9sZTG+dHVb8A43o9QaSCPngvHTZ4X9fx8lB96G7uju6pmm5ovKt85hxZW2uQNYo3ugwCUl94lVDYqo+KJUM="
-      python: 3.5
+      python: 3.7
       before_script: TOX_ENV=docs-travis
       # Note: graphviz is needed for grapgs embedded in the docs
       addons:
@@ -62,10 +72,10 @@ matrix:
         - python3 ./contrib/trigger_rtd_build.py
 
 install:
-  - pip install --upgrade "pip==19.1.1"
-  - pip install --upgrade "setuptools==41.6.0"
-  - pip install "virtualenv==16.7.7"
-  - pip install "tox==3.14.0"
+  - pip install --upgrade "pip==19.3.1"
+  - pip install --upgrade "setuptools==42.0.2"
+  - pip install "virtualenv==16.7.6"
+  - pip install "tox==3.14.2"
   - TOX_ENV=py$TRAVIS_PYTHON_VERSION
 
 script:

CHANGES.rst

@@ -1,11 +1,136 @@
 Changelog
 =========
 
-Changes in Apache Libcloud in development
------------------------------------------
+Changes in Apache Libcloud in development (3.0.0)
+-------------------------------------------------
+
+General
+~~~~~~~
+
+- This release drops support for Python versions older than 3.5.0.
+
+  If you still need to use Libcloud with Python 2.7 or Python 3.4 you can do
+  that by using the latest release which still supported those Python versions
+  (Libcloud v2.8.0).
+  (GITHUB-1377)
+  [Tomaz Muraus]
+
+- Make sure unit tests now also pass on Windows.
+  (GITHUB-1396)
+  [Tomaz Muraus]
+
+Compute
+~~~~~~~
+
+- [VMware vSphere] vSphere driver relies on ``pysphere`` Python library which
+  doesn't support Python 3 so it has been removed.
+
+  There is an unofficial ``pysphere`` fork which adds Python 3 support, but
+  it's out of date and not maintained (https://github.com/machalekj/pysphere/tree/2to3).
+  (GITHUB-1377)
+  [Tomaz Muraus]
+
+- [GCE] Fix ``ex_list_instancegroups`` method so it doesn't throw if ``zone``
+  attribute is not present in the response.
+
+  Reported by Kartik Subbarao (@kartiksubbarao)
+  (GITHUB-1346)
+  [Tomaz Muraus]
+
+- [AWS EC2] Add support for creating spot instances by utilizing new ``ex_spot``
+  and optionally also ``ex_spot_max_price`` keyword argument in the
+  ``create_node`` method.
+  (GITHUB-1398)
+  [Peter Yu - @yukw777]
+
+Storage
+~~~~~~~
+
+- [AWS S3] Fix upload object code so uploaded data MD5 checksum check is not
+  performed at the end of the upload when AWS KMS server side encryption is
+  used.
+
+  If AWS KMS server side object encryption is used, ETag header value in the
+  response doesn't contain data MD5 digest so we can't perform a checksum
+  check.
+
+  Reported by Jonathan Harden - @jfharden.
+  (GITHUB-1401, GITHUB-1406)
+  [Tomaz Muraus - @Kami]
+
+Container
+~~~~~~~~~
+
+- [LXD] Add new LXD driver.
+  (GITHUB-1395)
+  [Alexandros Giavaras - @pockerman]
+
+Storage
+~~~~~~~
+
+- [Azure Blobs] Implement chunked upload in the Azure Storage driver.
+
+  Previously, the maximum object size that could be uploaded with the
+  Azure Storage driver was capped at 100 MB: the maximum size that could
+  be uploaded in a single request to Azure. Chunked upload removes this
+  limitation and now enables uploading objects up to Azure's maximum block
+  blob size (~5 TB). The size of the chunks uploaded by the driver can be
+  configured via the ``LIBCLOUD_AZURE_UPLOAD_CHUNK_SIZE_MB`` environment
+  variable and defaults to 4 MB per chunk. Increasing this number trades-off
+  higher memory usage for a lower number of http requests executed by the
+  driver.
+
+  Reported by @rvolykh.
+  (GITHUB-1399, GITHUB-1400)
+  [Clemens Wolff - @c-w]
+
+- [Azure Blobs] Drop support for uploading PageBlob objects via the Azure
+  Storage driver.
+
+  Previously, both PageBlob and BlockBlob objects could be uploaded via the
+  ``upload_object`` and ``upload_object_via_stream`` methods by specifying the
+  ``ex_blob_type`` and ``ex_page_blob_size`` arguments. To simplify the API,
+  these options were removed and all uploaded objects are now of BlockBlob
+  type. Passing ``ex_blob_type`` or ``ex_page_blob_size`` will now raise a
+  ``ValueError``.
+
+  (GITHUB-1400)
+  [Clemens Wolff - @c-w]
+
+- [Common] Add ``prefix`` argument to ``iterate_container_objects`` and
+  ``list_container_objects`` to support object-list filtering in all
+  StorageDriver implementations.
+
+  A lot of the existing storage drivers already implemented the filtering
+  functionality via the ``ex_prefix`` extension argument so it was decided
+  to promote the argument to be part of the standard Libcloud storage API.
+  For any storage driver that doesn't natively implement filtering the results
+  list, a fall-back was implemented which filters the full object stream on
+  the client side.
+
+  For backward compatibility reasons, the ``ex_prefix`` argument will still
+  be respected until a next major release.
+  (GITHUB-1397)
+  [Clemens Wolff - @c-w]
+
+- [Azure Blobs] Implement ``get_object_cdn_url`` for the Azure Storage driver.
+
+  Leveraging Azure storage service shared access signatures, the Azure Storage
+  driver can now be used to generate temporary URLs that grant clients read
+  access to objects. The URLs expire after a certain period of time, either
+  configured via the ``ex_expiry`` argument or the
+  ``LIBCLOUD_AZURE_STORAGE_CDN_URL_EXPIRY_HOURS`` environment variable
+  (default: 24 hours).
+
+  Reported by @rvolykh.
+  (GITHUB-1403, GITHUB-1408)
+  [Clemens Wolff - @c-w]
+
+Changes in Apache Libcloud v2.8.0
+---------------------------------
 
 Common
-------
+~~~~~~
 
 - Fix a regression with ``get_driver()`` method not working if ``provider``
   argument value was a string (e.g. using ``get_driver('openstack')``
@@ -35,7 +160,7 @@ Common
   [Tomaz Muraus]
 
 Compute
--------
+~~~~~~~
 
 - [DigitalOcean] Fix ``attach_volume`` and ``detach_volume`` methods.
   Previously those two methods incorrectly passed volume id instead of
@@ -92,7 +217,7 @@ Compute
   [Tomaz Muraus]
 
 Storage
--------
+~~~~~~~
 
 - [AWS S3] Make sure ``host`` driver constructor argument has priority
   over ``region`` argument.
@@ -107,14 +232,14 @@ Changes in Apache Libcloud v2.7.0
 ---------------------------------
 
 General
--------
+~~~~~~~
 
 - Test code with Python 3.8 and advertise that we also support Python 3.8.
   (GITHUB-1371, GITHUB-1374)
   [Tomaz Muraus]
 
 Common
-------
+~~~~~~
 
 - [OpenStack] Fix OpenStack project scoped token authentication. The driver
   constructors now accept ``ex_tenant_domain_id`` argument which tells
@@ -123,7 +248,7 @@ Common
   [kshtsk]
 
 Compute
--------
+~~~~~~~
 
 - Introduce type annotations for the base compute API methods. This means you
   can now leverage mypy to type check (with some limitations) your code which

contrib/generate_contributor_list.py

@@ -112,7 +112,7 @@ def convert_to_markdown(contributors_map, include_tickets=False):
     def compare(item1, item2):
         lastname1 = item1.split(' ')[-1].lower()
         lastname2 = item2.split(' ')[-1].lower()
-        return cmp(lastname1, lastname2)
+        return (lastname1 > lastname2) - (lastname1 < lastname2)
 
     names = contributors_map.keys()
     names = sorted(names, cmp=compare)

contrib/generate_provider_logos_collage_image.py

@@ -68,7 +68,7 @@ def resize_images(logo_files, output_path):
         new_name = '%s%s' % (name, ext)
         out_name = pjoin(output_path, 'resized/', new_name)
 
-        print 'Resizing image: %(name)s' % {'name': logo_file}
+        print('Resizing image: %(name)s' % {'name': logo_file})
 
         values = {'name': logo_file, 'out_name': out_name,
                   'dimensions': DIMENSIONS}
@@ -89,7 +89,7 @@ def assemble_final_image(resized_images, output_path):
     cmd = 'montage %(images)s -geometry %(geometry)s %(out_name)s'
     cmd = cmd % values
 
-    print 'Generating final image: %(name)s' % {'name': final_name}
+    print('Generating final image: %(name)s' % {'name': final_name})
     subprocess.call(cmd, shell=True)
 
 

contrib/update_google_prices.py

@@ -18,6 +18,8 @@
 Loads Google Cloud Platform prices and updates the `pricing.json` data file.
 """
 
+# pylint: skip-file
+
 import os
 import json
 import sys

contrib/utils_test.py

@@ -16,6 +16,8 @@
 #
 ################################################################################
 
+# pylint: skip-file
+
 import unittest
 import utils
 

demos/example_aliyun_oss.py

@@ -36,7 +36,7 @@
 count = len(objects)
 print('Has %d objects' % count)
 
-objects = oss.list_container_objects(c1, ex_prefix='en')
+objects = oss.list_container_objects(c1, prefix='en')
 print('Has %d objects with prefix "en"' % len(objects))
 for each in objects:
     print(each)

demos/gce_demo.py

@@ -81,7 +81,7 @@
     # pylint: disable=no-name-in-module,import-error
     import urllib.request as url_req
 else:
-    import urllib2 as url_req
+    import urllib2 as url_req  # pylint: disable=import-error
 
 # Maximum number of 1-CPU nodes to allow to run simultaneously
 MAX_NODES = 5

doap_libcloud.rdf

@@ -420,6 +420,13 @@
         <revision>v2.7.0</revision>
       </Version>
   </release>
+  <release>
+      <Version>
+        <name>2.8.0</name>
+        <created>2020-01-02</created>
+        <revision>v2.8.0</revision>
+      </Version>
+  </release>
   <repository>
       <SVNRepository>
         <location rdf:resource="https://svn.apache.org/repos/asf/libcloud/trunk/"/>

docs/compute/_supported_methods_block_storage.rst

@@ -63,7 +63,6 @@ Provider                              list volumes create volume destroy volume
 `vCloud`_                             no           no            no             no            no            no             no             
 `Voxel VoxCLOUD`_                     no           no            no             no            no            no             no             
 `vps.net`_                            no           no            no             no            no            no             no             
-`VMware vSphere`_                     no           no            no             no            no            no             no             
 `Vultr`_                              no           no            no             no            no            no             no             
 ===================================== ============ ============= ============== ============= ============= ============== ===============
 
@@ -127,5 +126,4 @@ Provider                              list volumes create volume destroy volume
 .. _`vCloud`: http://www.vmware.com/products/vcloud/
 .. _`Voxel VoxCLOUD`: http://www.voxel.net/
 .. _`vps.net`: http://vps.net/
-.. _`VMware vSphere`: http://www.vmware.com/products/vsphere/
 .. _`Vultr`: https://www.vultr.com