Generate code for Bigtable v2 protobufs

.coveragerc

@@ -1,6 +1,7 @@
 [report]
 omit =
     */_generated/*.py
+    */_generated_v2/*.py
 show_missing = True
 exclude_lines =
     # Re-enable the standard pragma

Makefile → Makefile.bigtable_v1

@@ -1,9 +1,10 @@
 GENERATED_DIR=$(shell pwd)/generated_python
-BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/_generated
-DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
+GENERATED_SUBDIR=_generated
+BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
 GRPC_PLUGIN=grpc_python_plugin
 PROTOC_CMD=protoc
-BIGTABLE_PROTOS_DIR=$(shell pwd)/cloud-bigtable-client/bigtable-protos/src/main/proto
+BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
+BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
 GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
 
 help:
@@ -15,8 +16,8 @@ help:
 
 generate:
 	# Retrieve git repos that have our *.proto files.
-	[ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
-	cd cloud-bigtable-client && git pull origin master
+	[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
+	cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
 	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
 	cd googleapis-pb && git pull origin master
 	# Make the directory where our *_pb2.py files will go.
@@ -41,49 +42,37 @@ generate:
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_data.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto \
-	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/datastore.proto \
-	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/entity.proto \
-	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/query.proto
 	# Move the newly generated *_pb2.py files into our library.
-	mv $(GENERATED_DIR)/google/bigtable/v1/* $(BIGTABLE_DIR)
-	mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(BIGTABLE_DIR)
-	mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(BIGTABLE_DIR)
-	mv $(GENERATED_DIR)/google/datastore/v1beta3/* $(DATASTORE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/v1/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(BIGTABLE_DIR)
 	# Remove all existing *.proto files before we replace
 	rm -f $(BIGTABLE_DIR)/*.proto
-	rm -f $(DATASTORE_DIR)/*.proto
 	# Copy over the *.proto files into our library.
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
-	cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/*.proto $(DATASTORE_DIR)
 	# Rename all *.proto files in our library with an
 	# underscore and remove executable bit.
 	cd $(BIGTABLE_DIR) && \
 	    for filename in *.proto; do \
 	        chmod -x $$filename ; \
 	        mv $$filename _$$filename ; \
 	    done
-	cd $(DATASTORE_DIR) && \
-	    for filename in *.proto; do \
-	        chmod -x $$filename ; \
-	        mv $$filename _$$filename ; \
-	    done
 	# Separate the gRPC parts of the operations service from the
 	# non-gRPC parts so that the protos from `googleapis-common-protos`
 	# can be used without gRPC.
-	python scripts/make_operations_grpc.py
-	# Separate the gRPC parts of the datastore service from the
-	# non-gRPC parts so that the protos can be used without gRPC.
-	python scripts/make_datastore_grpc.py
+	PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
+		GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+		python scripts/make_operations_grpc.py
 	# Rewrite the imports in the generated *_pb2.py files.
-	python scripts/rewrite_imports.py
+	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
 
 check_generate:
 	python scripts/check_generate.py
 
 clean:
-	rm -fr cloud-bigtable-client $(GENERATED_DIR)
+	rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
 
 .PHONY: generate check_generate clean

Makefile.bigtable_v2

@@ -0,0 +1,75 @@
+GENERATED_DIR=$(shell pwd)/generated_python
+GENERATED_SUBDIR=_generated_v2
+BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
+GRPC_PLUGIN=grpc_python_plugin
+PROTOC_CMD=protoc
+BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
+BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
+GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
+
+help:
+	@echo 'Makefile for gcloud-python Bigtable protos                      '
+	@echo '                                                                '
+	@echo '   make generate                 Generates the protobuf modules '
+	@echo '   make check_generate           Checks that generate succeeded '
+	@echo '   make clean                    Clean generated files          '
+
+generate:
+	# Retrieve git repos that have our *.proto files.
+	[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
+	cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
+	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
+	cd googleapis-pb && git pull origin master
+	# Make the directory where our *_pb2.py files will go.
+	mkdir -p $(GENERATED_DIR)
+	# Generate all *_pb2.py files that require gRPC.
+	$(PROTOC_CMD) \
+	    --proto_path=$(BIGTABLE_PROTOS_DIR) \
+	    --python_out=$(GENERATED_DIR) \
+	    --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
+	    --grpc_out=$(GENERATED_DIR) \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
+	# Generate all *_pb2.py files that do not require gRPC.
+	$(PROTOC_CMD) \
+	    --proto_path=$(BIGTABLE_PROTOS_DIR) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
+	    --python_out=$(GENERATED_DIR) \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/data.proto \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
+	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
+	# Move the newly generated *_pb2.py files into our library.
+	cp $(GENERATED_DIR)/google/bigtable/v2/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
+	cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
+	# Remove all existing *.proto files before we replace
+	rm -f $(BIGTABLE_DIR)/*.proto
+	# Copy over the *.proto files into our library.
+	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
+	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
+	cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
+	# Rename all *.proto files in our library with an
+	# underscore and remove executable bit.
+	cd $(BIGTABLE_DIR) && \
+	    for filename in *.proto; do \
+	        chmod -x $$filename ; \
+	        mv $$filename _$$filename ; \
+	    done
+	# Separate the gRPC parts of the operations service from the
+	# non-gRPC parts so that the protos from `googleapis-common-protos`
+	# can be used without gRPC.
+	PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
+		GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+		python scripts/make_operations_grpc.py
+	# Rewrite the imports in the generated *_pb2.py files.
+	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
+
+check_generate:
+	python scripts/check_generate.py
+
+clean:
+	rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
+
+.PHONY: generate check_generate clean

Makefile.datastore

@@ -0,0 +1,53 @@
+GENERATED_DIR=$(shell pwd)/generated_python
+DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
+GRPC_PLUGIN=grpc_python_plugin
+PROTOC_CMD=protoc
+GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
+
+help:
+	@echo 'Makefile for gcloud-python Bigtable protos                      '
+	@echo '                                                                '
+	@echo '   make generate                 Generates the protobuf modules '
+	@echo '   make check_generate           Checks that generate succeeded '
+	@echo '   make clean                    Clean generated files          '
+
+generate:
+	# Retrieve git repos that have our *.proto files.
+	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
+	cd googleapis-pb && git pull origin master
+	# Make the directory where our *_pb2.py files will go.
+	mkdir -p $(GENERATED_DIR)
+	# Generate all *_pb2.py files that do not require gRPC.
+	$(PROTOC_CMD) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
+	    --python_out=$(GENERATED_DIR) \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/datastore.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/entity.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/query.proto
+	# Move the newly generated *_pb2.py files into our library.
+	cp $(GENERATED_DIR)/google/datastore/v1beta3/* $(DATASTORE_DIR)
+	# Remove all existing *.proto files before we replace
+	rm -f $(DATASTORE_DIR)/*.proto
+	# Copy over the *.proto files into our library.
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/*.proto $(DATASTORE_DIR)
+	# Rename all *.proto files in our library with an
+	# underscore and remove executable bit.
+	cd $(DATASTORE_DIR) && \
+	    for filename in *.proto; do \
+	        chmod -x $$filename ; \
+	        mv $$filename _$$filename ; \
+	    done
+	# Separate the gRPC parts of the datastore service from the
+	# non-gRPC parts so that the protos can be used without gRPC.
+	PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
+		python scripts/make_datastore_grpc.py
+	# Rewrite the imports in the generated *_pb2.py files.
+	python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py
+
+check_generate:
+	python scripts/check_generate.py
+
+clean:
+	rm -fr $(GENERATED_DIR)
+
+.PHONY: generate check_generate clean

gcloud/bigtable/_generated_v2/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generated protobuf modules for Google Cloud Bigtable API."""