Support blob streaming for file-like objects

googleapis · Jul 14, 2016 · 742a95e · 742a95e
1 parent 942c5d0
commit 742a95e
Show file tree

Hide file tree

Showing 111 changed files with 13,059 additions and 2,991 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -1,6 +1,7 @@
 [report]
 omit =
     */_generated/*.py
+    */_generated_v2/*.py
 show_missing = True
 exclude_lines =
     # Re-enable the standard pragma

diff --git a/.gitignore b/.gitignore
@@ -57,3 +57,4 @@ scripts/pylintrc_reduced
 generated_python/
 cloud-bigtable-client/
 googleapis-pb/
+grpc_python_venv/
diff --git a/.travis.yml b/.travis.yml
@@ -27,7 +27,8 @@ deploy:
     repo: GoogleCloudPlatform/gcloud-python
     # until this is fixed: https://github.com/travis-ci/travis-ci/issues/1675
     all_branches: true
-  distributions: "sdist bdist_wheel"
+  # 'bdist_wheel' builds disabled until #1879 et al. are resolved.
+  distributions: "sdist"
 
 cache:
   directories:

diff --git a/LICENSE b/LICENSE
@@ -1,4 +1,5 @@
-Apache License
+
+                                 Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
@@ -178,7 +179,7 @@ Apache License
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -199,4 +200,3 @@ Apache License
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-
diff --git a/Makefile → Makefile.bigtable_v1 b/Makefile → Makefile.bigtable_v1
@@ -1,10 +1,11 @@
+GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
 GENERATED_DIR=$(shell pwd)/generated_python
-BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/_generated
-DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
-GRPC_PLUGIN=grpc_python_plugin
-PROTOC_CMD=protoc
-BIGTABLE_PROTOS_DIR=$(shell pwd)/cloud-bigtable-client/bigtable-protos/src/main/proto
+GENERATED_SUBDIR=_generated
+BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
+PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
 GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
+BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
+BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-client-core-parent/bigtable-protos/src/main/proto
 
 help:
 	@echo 'Makefile for gcloud-python Bigtable protos                      '
@@ -14,19 +15,22 @@ help:
 	@echo '   make clean                    Clean generated files          '
 
 generate:
+	# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
+	[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
+	$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
 	# Retrieve git repos that have our *.proto files.
-	[ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
-	cd cloud-bigtable-client && git pull origin master
-	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
-	cd googleapis-pb && git pull origin master
+	[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
+	cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
+	[ -d $(GOOGLEAPIS_PROTOS_DIR) ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
+	cd $(GOOGLEAPIS_PROTOS_DIR) && git pull origin master
 	# Make the directory where our *_pb2.py files will go.
 	mkdir -p $(GENERATED_DIR)
 	# Generate all *_pb2.py files that require gRPC.
 	$(PROTOC_CMD) \
 	    --proto_path=$(BIGTABLE_PROTOS_DIR) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
 	    --python_out=$(GENERATED_DIR) \
-	    --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
-	    --grpc_out=$(GENERATED_DIR) \
+	    --grpc_python_out=$(GENERATED_DIR) \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto
@@ -41,49 +45,37 @@ generate:
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_data.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto \
-	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/datastore.proto \
-	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/entity.proto \
-	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/query.proto
 	# Move the newly generated *_pb2.py files into our library.
-	mv $(GENERATED_DIR)/google/bigtable/v1/* $(BIGTABLE_DIR)
-	mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(BIGTABLE_DIR)
-	mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(BIGTABLE_DIR)
-	mv $(GENERATED_DIR)/google/datastore/v1beta3/* $(DATASTORE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/v1/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(BIGTABLE_DIR)
 	# Remove all existing *.proto files before we replace
 	rm -f $(BIGTABLE_DIR)/*.proto
-	rm -f $(DATASTORE_DIR)/*.proto
 	# Copy over the *.proto files into our library.
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(BIGTABLE_DIR)
-	cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
-	cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/*.proto $(DATASTORE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
 	# Rename all *.proto files in our library with an
 	# underscore and remove executable bit.
 	cd $(BIGTABLE_DIR) && \
 	    for filename in *.proto; do \
 	        chmod -x $$filename ; \
 	        mv $$filename _$$filename ; \
 	    done
-	cd $(DATASTORE_DIR) && \
-	    for filename in *.proto; do \
-	        chmod -x $$filename ; \
-	        mv $$filename _$$filename ; \
-	    done
 	# Separate the gRPC parts of the operations service from the
 	# non-gRPC parts so that the protos from `googleapis-common-protos`
 	# can be used without gRPC.
-	python scripts/make_operations_grpc.py
-	# Separate the gRPC parts of the datastore service from the
-	# non-gRPC parts so that the protos can be used without gRPC.
-	python scripts/make_datastore_grpc.py
+	GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
+	GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+		python scripts/make_operations_grpc.py
 	# Rewrite the imports in the generated *_pb2.py files.
-	python scripts/rewrite_imports.py
+	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
 
 check_generate:
 	python scripts/check_generate.py
 
 clean:
-	rm -fr cloud-bigtable-client $(GENERATED_DIR)
+	rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
 
 .PHONY: generate check_generate clean
diff --git a/Makefile.bigtable_v2 b/Makefile.bigtable_v2
@@ -0,0 +1,71 @@
+GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
+GENERATED_DIR=$(shell pwd)/generated_python
+GENERATED_SUBDIR=_generated_v2
+BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
+PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
+GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
+
+help:
+	@echo 'Makefile for gcloud-python Bigtable protos                      '
+	@echo '                                                                '
+	@echo '   make generate                 Generates the protobuf modules '
+	@echo '   make check_generate           Checks that generate succeeded '
+	@echo '   make clean                    Clean generated files          '
+
+generate:
+	# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
+	[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
+	$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
+	# Retrieve git repos that have our *.proto files.
+	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
+	cd googleapis-pb && git pull origin master
+	# Make the directory where our *_pb2.py files will go.
+	mkdir -p $(GENERATED_DIR)
+	# Generate all *_pb2.py files that require gRPC.
+	$(PROTOC_CMD) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
+	    --python_out=$(GENERATED_DIR) \
+	    --grpc_python_out=$(GENERATED_DIR) \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
+	# Generate all *_pb2.py files that do not require gRPC.
+	$(PROTOC_CMD) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
+	    --python_out=$(GENERATED_DIR) \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/data.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
+	# Move the newly generated *_pb2.py files into our library.
+	cp $(GENERATED_DIR)/google/bigtable/v2/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
+	# Remove all existing *.proto files before we replace
+	rm -f $(BIGTABLE_DIR)/*.proto
+	# Copy over the *.proto files into our library.
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
+	# Rename all *.proto files in our library with an
+	# underscore and remove executable bit.
+	cd $(BIGTABLE_DIR) && \
+	    for filename in *.proto; do \
+	        chmod -x $$filename ; \
+	        mv $$filename _$$filename ; \
+	    done
+	# Separate the gRPC parts of the operations service from the
+	# non-gRPC parts so that the protos from `googleapis-common-protos`
+	# can be used without gRPC.
+	GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
+	GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+		python scripts/make_operations_grpc.py
+	# Rewrite the imports in the generated *_pb2.py files.
+	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
+
+check_generate:
+	python scripts/check_generate.py
+
+clean:
+	rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
+
+.PHONY: generate check_generate clean
diff --git a/Makefile.datastore b/Makefile.datastore
@@ -0,0 +1,57 @@
+GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
+GENERATED_DIR=$(shell pwd)/generated_python
+DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
+PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
+GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
+
+help:
+	@echo 'Makefile for gcloud-python Bigtable protos                      '
+	@echo '                                                                '
+	@echo '   make generate                 Generates the protobuf modules '
+	@echo '   make check_generate           Checks that generate succeeded '
+	@echo '   make clean                    Clean generated files          '
+
+generate:
+	# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
+	[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
+	$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
+	# Retrieve git repos that have our *.proto files.
+	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
+	cd googleapis-pb && git pull origin master
+	# Make the directory where our *_pb2.py files will go.
+	mkdir -p $(GENERATED_DIR)
+	# Generate all *_pb2.py files that do not require gRPC.
+	$(PROTOC_CMD) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
+	    --python_out=$(GENERATED_DIR) \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/datastore.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/entity.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/query.proto
+	# Move the newly generated *_pb2.py files into our library.
+	cp $(GENERATED_DIR)/google/datastore/v1beta3/* $(DATASTORE_DIR)
+	# Remove all existing *.proto files before we replace
+	rm -f $(DATASTORE_DIR)/*.proto
+	# Copy over the *.proto files into our library.
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/*.proto $(DATASTORE_DIR)
+	# Rename all *.proto files in our library with an
+	# underscore and remove executable bit.
+	cd $(DATASTORE_DIR) && \
+	    for filename in *.proto; do \
+	        chmod -x $$filename ; \
+	        mv $$filename _$$filename ; \
+	    done
+	# Separate the gRPC parts of the datastore service from the
+	# non-gRPC parts so that the protos can be used without gRPC.
+	GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
+	GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+		python scripts/make_datastore_grpc.py
+	# Rewrite the imports in the generated *_pb2.py files.
+	python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py
+
+check_generate:
+	python scripts/check_generate.py
+
+clean:
+	rm -fr $(GENERATED_DIR)
+
+.PHONY: generate check_generate clean
diff --git a/README.rst b/README.rst
@@ -20,12 +20,14 @@ This client supports the following Google Cloud Platform services:
 -  `Google Cloud Pub/Sub`_
 -  `Google BigQuery`_
 -  `Google Cloud Resource Manager`_
+-  `Google Stackdriver Logging`_
 
 .. _Google Cloud Datastore: https://github.com/GoogleCloudPlatform/gcloud-python#google-cloud-datastore
 .. _Google Cloud Storage: https://github.com/GoogleCloudPlatform/gcloud-python#google-cloud-storage
 .. _Google Cloud Pub/Sub: https://github.com/GoogleCloudPlatform/gcloud-python#google-cloud-pubsub
 .. _Google BigQuery: https://github.com/GoogleCloudPlatform/gcloud-python#google-bigquery
 .. _Google Cloud Resource Manager: https://github.com/GoogleCloudPlatform/gcloud-python#google-cloud-resource-manager
+.. _Google Stackdriver Logging: https://github.com/GoogleCloudPlatform/gcloud-python#google-cloud-logging
 
 If you need support for other Google APIs, check out the
 `Google APIs Python Client library`_.
@@ -174,6 +176,49 @@ append-only tables, using the processing power of Google's infrastructure.
 
 This package is still being implemented, but it is almost complete!
 
+Load data from CSV
+~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+    import csv
+
+    from gcloud import bigquery
+    from gcloud.bigquery import SchemaField
+
+    client = bigquery.Client()
+
+    dataset = client.dataset('dataset_name')
+    dataset.create()  # API request
+
+    SCHEMA = [
+        SchemaField('full_name', 'STRING', mode='required'),
+        SchemaField('age', 'INTEGER', mode='required'),
+    ]
+    table = dataset.table('table_name', SCHEMA)
+    table.create()
+
+    with open('csv_file', 'rb') as readable:
+        table.upload_from_file(
+            readable, source_format='CSV', skip_leading_rows=1)
+
+Perform a synchronous query
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: python
+
+    # Perform a synchronous query.
+    QUERY = (
+        'SELECT name FROM [bigquery-public-data:usa_names.usa_1910_2013] '
+        'WHERE state = "TX"')
+    query = client.run_sync_query('%s LIMIT 100' % QUERY)
+    query.timeout_ms = TIMEOUT_MS
+    query.run()
+
+    for row in query.rows:
+        print row
+
+
 See the ``gcloud-python`` API `BigQuery documentation`_ to learn how to connect
 to BigQuery using this Client Library.
 
@@ -194,6 +239,35 @@ manage projects using this Client Library.
 
 .. _Resource Manager documentation: https://googlecloudplatform.github.io/gcloud-python/stable/resource-manager-api.html
 
+Google Stackdriver Logging
+--------------------------
+
+`Stackdriver Logging`_ API (`Logging API docs`_) allows you to store, search,
+analyze, monitor, and alert on log data and events from Google Cloud Platform.
+
+.. _Stackdriver Logging: https://cloud.google.com/logging/
+.. _Logging API docs: https://cloud.google.com/logging/docs/
+
+.. code:: python
+
+    from gcloud import logging
+    client = logging.Client()
+    logger = client.logger('log_name')
+    logger.log_text("A simple entry")  # API call
+
+Example of fetching entries:
+
+.. code:: python
+
+    entries, token = logger.list_entries()
+    for entry in entries:
+        print entry.payload
+
+See the ``gcloud-python`` API `logging documentation`_ to learn how to connect
+to Stackdriver Logging using this Client Library.
+
+.. _logging documentation: https://googlecloudplatform.github.io/gcloud-python/stable/logging-usage.html
+
 Contributing
 ------------