Skip to content

Commit e570b03

Browse files
tseavertseaver
authored
Merge pull request #1903 from tseaver/bigtable-v2-regenerate_protobufs
Generate code for Bigtable v2 protobufs
2 parents 986302f + 9d7d8af commit e570b03

26 files changed

+5887
-35
lines changed

‎.coveragerc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
[report]
22
omit =
33
*/_generated/*.py
4+
*/_generated_v2/*.py
45
show_missing = True
56
exclude_lines =
67
# Re-enable the standard pragma

‎Makefile‎ renamed to ‎Makefile.bigtable_v1‎

Lines changed: 14 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
GENERATED_DIR=$(shell pwd)/generated_python
2-
BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/_generated
3-
DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
2+
GENERATED_SUBDIR=_generated
3+
BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
44
GRPC_PLUGIN=grpc_python_plugin
55
PROTOC_CMD=protoc
6-
BIGTABLE_PROTOS_DIR=$(shell pwd)/cloud-bigtable-client/bigtable-protos/src/main/proto
6+
BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
7+
BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
78
GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
89

910
help:
@@ -15,8 +16,8 @@ help:
1516

1617
generate:
1718
# Retrieve git repos that have our *.proto files.
18-
[ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
19-
cd cloud-bigtable-client && git pull origin master
19+
[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
20+
cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
2021
[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
2122
cd googleapis-pb && git pull origin master
2223
# Make the directory where our *_pb2.py files will go.
@@ -41,49 +42,37 @@ generate:
4142
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service_messages.proto \
4243
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_data.proto \
4344
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service_messages.proto \
44-
$(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/datastore.proto \
45-
$(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/entity.proto \
46-
$(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/query.proto
4745
# Move the newly generated *_pb2.py files into our library.
48-
mv $(GENERATED_DIR)/google/bigtable/v1/* $(BIGTABLE_DIR)
49-
mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(BIGTABLE_DIR)
50-
mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(BIGTABLE_DIR)
51-
mv $(GENERATED_DIR)/google/datastore/v1beta3/* $(DATASTORE_DIR)
46+
cp $(GENERATED_DIR)/google/bigtable/v1/* $(BIGTABLE_DIR)
47+
cp $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(BIGTABLE_DIR)
48+
cp $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(BIGTABLE_DIR)
5249
# Remove all existing *.proto files before we replace
5350
rm -f $(BIGTABLE_DIR)/*.proto
54-
rm -f $(DATASTORE_DIR)/*.proto
5551
# Copy over the *.proto files into our library.
5652
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/*.proto $(BIGTABLE_DIR)
5753
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(BIGTABLE_DIR)
5854
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(BIGTABLE_DIR)
5955
cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
60-
cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/*.proto $(DATASTORE_DIR)
6156
# Rename all *.proto files in our library with an
6257
# underscore and remove executable bit.
6358
cd $(BIGTABLE_DIR) && \
6459
for filename in *.proto; do \
6560
chmod -x $$filename ; \
6661
mv $$filename _$$filename ; \
6762
done
68-
cd $(DATASTORE_DIR) && \
69-
for filename in *.proto; do \
70-
chmod -x $$filename ; \
71-
mv $$filename _$$filename ; \
72-
done
7363
# Separate the gRPC parts of the operations service from the
7464
# non-gRPC parts so that the protos from `googleapis-common-protos`
7565
# can be used without gRPC.
76-
python scripts/make_operations_grpc.py
77-
# Separate the gRPC parts of the datastore service from the
78-
# non-gRPC parts so that the protos can be used without gRPC.
79-
python scripts/make_datastore_grpc.py
66+
PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
67+
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
68+
python scripts/make_operations_grpc.py
8069
# Rewrite the imports in the generated *_pb2.py files.
81-
python scripts/rewrite_imports.py
70+
python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
8271

8372
check_generate:
8473
python scripts/check_generate.py
8574

8675
clean:
87-
rm -fr cloud-bigtable-client $(GENERATED_DIR)
76+
rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
8877

8978
.PHONY: generate check_generate clean

‎Makefile.bigtable_v2‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
GENERATED_DIR=$(shell pwd)/generated_python
2+
GENERATED_SUBDIR=_generated_v2
3+
BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
4+
GRPC_PLUGIN=grpc_python_plugin
5+
PROTOC_CMD=protoc
6+
BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
7+
BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
8+
GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
9+
10+
help:
11+
@echo 'Makefile for gcloud-python Bigtable protos '
12+
@echo ' '
13+
@echo ' make generate Generates the protobuf modules '
14+
@echo ' make check_generate Checks that generate succeeded '
15+
@echo ' make clean Clean generated files '
16+
17+
generate:
18+
# Retrieve git repos that have our *.proto files.
19+
[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
20+
cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
21+
[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
22+
cd googleapis-pb && git pull origin master
23+
# Make the directory where our *_pb2.py files will go.
24+
mkdir -p $(GENERATED_DIR)
25+
# Generate all *_pb2.py files that require gRPC.
26+
$(PROTOC_CMD) \
27+
--proto_path=$(BIGTABLE_PROTOS_DIR) \
28+
--python_out=$(GENERATED_DIR) \
29+
--plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
30+
--grpc_out=$(GENERATED_DIR) \
31+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
32+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
33+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
34+
# Generate all *_pb2.py files that do not require gRPC.
35+
$(PROTOC_CMD) \
36+
--proto_path=$(BIGTABLE_PROTOS_DIR) \
37+
--proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
38+
--python_out=$(GENERATED_DIR) \
39+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/data.proto \
40+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
41+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
42+
$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
43+
# Move the newly generated *_pb2.py files into our library.
44+
cp $(GENERATED_DIR)/google/bigtable/v2/* $(BIGTABLE_DIR)
45+
cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
46+
cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
47+
# Remove all existing *.proto files before we replace
48+
rm -f $(BIGTABLE_DIR)/*.proto
49+
# Copy over the *.proto files into our library.
50+
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
51+
cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
52+
cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
53+
# Rename all *.proto files in our library with an
54+
# underscore and remove executable bit.
55+
cd $(BIGTABLE_DIR) && \
56+
for filename in *.proto; do \
57+
chmod -x $$filename ; \
58+
mv $$filename _$$filename ; \
59+
done
60+
# Separate the gRPC parts of the operations service from the
61+
# non-gRPC parts so that the protos from `googleapis-common-protos`
62+
# can be used without gRPC.
63+
PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
64+
GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
65+
python scripts/make_operations_grpc.py
66+
# Rewrite the imports in the generated *_pb2.py files.
67+
python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
68+
69+
check_generate:
70+
python scripts/check_generate.py
71+
72+
clean:
73+
rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
74+
75+
.PHONY: generate check_generate clean

‎Makefile.datastore‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
GENERATED_DIR=$(shell pwd)/generated_python
2+
DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
3+
GRPC_PLUGIN=grpc_python_plugin
4+
PROTOC_CMD=protoc
5+
GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
6+
7+
help:
8+
@echo 'Makefile for gcloud-python Bigtable protos '
9+
@echo ' '
10+
@echo ' make generate Generates the protobuf modules '
11+
@echo ' make check_generate Checks that generate succeeded '
12+
@echo ' make clean Clean generated files '
13+
14+
generate:
15+
# Retrieve git repos that have our *.proto files.
16+
[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
17+
cd googleapis-pb && git pull origin master
18+
# Make the directory where our *_pb2.py files will go.
19+
mkdir -p $(GENERATED_DIR)
20+
# Generate all *_pb2.py files that do not require gRPC.
21+
$(PROTOC_CMD) \
22+
--proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
23+
--python_out=$(GENERATED_DIR) \
24+
$(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/datastore.proto \
25+
$(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/entity.proto \
26+
$(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/query.proto
27+
# Move the newly generated *_pb2.py files into our library.
28+
cp $(GENERATED_DIR)/google/datastore/v1beta3/* $(DATASTORE_DIR)
29+
# Remove all existing *.proto files before we replace
30+
rm -f $(DATASTORE_DIR)/*.proto
31+
# Copy over the *.proto files into our library.
32+
cp $(GOOGLEAPIS_PROTOS_DIR)/google/datastore/v1beta3/*.proto $(DATASTORE_DIR)
33+
# Rename all *.proto files in our library with an
34+
# underscore and remove executable bit.
35+
cd $(DATASTORE_DIR) && \
36+
for filename in *.proto; do \
37+
chmod -x $$filename ; \
38+
mv $$filename _$$filename ; \
39+
done
40+
# Separate the gRPC parts of the datastore service from the
41+
# non-gRPC parts so that the protos can be used without gRPC.
42+
PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
43+
python scripts/make_datastore_grpc.py
44+
# Rewrite the imports in the generated *_pb2.py files.
45+
python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py
46+
47+
check_generate:
48+
python scripts/check_generate.py
49+
50+
clean:
51+
rm -fr $(GENERATED_DIR)
52+
53+
.PHONY: generate check_generate clean

‎gcloud/bigtable/_generated_v2/__init__.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2015 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Generated protobuf modules for Google Cloud Bigtable API."""

0 commit comments

Comments
 (0)