Skip to content

Commit 0d8f611

Browse files
kongweihankongweihan
authored
docs: Add an example for batch write flow control (GoogleCloudPlatform#9314)
Includes using BigtableIO and CloudBigtableIO
1 parent 3bd50b1 commit 0d8f611

File tree

4 files changed

+536
-0
lines changed

4 files changed

+536
-0
lines changed

‎bigtable/beam/batch-write-flow-control-example/README.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Batch write flow control example
2+
3+
This is an example pipeline to demo how to use the batch write flow control
4+
feature using CloudBigtableIO and BigtableIO.
5+
6+
## Running instructions
7+
8+
1. Create a Bigtable instance in the console or using gCloud.
9+
10+
1. Create a table with column family `cf`.
11+
12+
1. Set up the environment variables
13+
14+
```
15+
GOOGLE_CLOUD_PROJECT=<your-project-id>
16+
INSTANCE_ID=<your-instance-id>
17+
TABLE_ID=<your-table-id>
18+
REGION=<your-region>
19+
NUM_ROWS=<number-of-rows>
20+
NUM_COLS_PER_ROW=<number-of-columns-per-row>
21+
NUM_BYTES_PER_COL=<number-of-bytes-per-col>
22+
NUM_WORKERS=<number-of-workers>
23+
MAX_NUM_WORKERS=<max-number-of-workers>
24+
USE_CLOUD_BIGTABLE_IO=<true/false>
25+
26+
```
27+
28+
1. Run the command
29+
30+
```
31+
mvn compile exec:java -Dexec.mainClass=bigtable.BatchWriteFlowControlExample \
32+
"-Dexec.args=--runner=dataflow \
33+
--project=$GOOGLE_CLOUD_PROJECT \
34+
--bigtableInstanceId=$INSTANCE_ID \
35+
--bigtableTableId=$TABLE_ID \
36+
--bigtableRows=$NUM_ROWS \
37+
--bigtableColsPerRow=$NUM_COLS_PER_ROW \
38+
--bigtableBytesPerCol=$NUM_BYTES_PER_COL\
39+
--region=$REGION \
40+
--numWorkers=$NUM_WORKERS \
41+
--maxNumWorkers=$MAX_NUM_WORKERS \
42+
--useCloudBigtableIo=$USE_CLOUD_BIGTABLE_IO"
43+
```

‎bigtable/beam/batch-write-flow-control-example/pom.xml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright 2024 Google LLC
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
22+
<groupId>com.example.bigtable</groupId>
23+
<artifactId>batch-write-flow-control-example</artifactId>
24+
<version>1.0-SNAPSHOT</version>
25+
26+
<properties>
27+
<maven.compiler.source>1.8</maven.compiler.source>
28+
<maven.compiler.target>1.8</maven.compiler.target>
29+
<apache_beam.version>2.56.0</apache_beam.version>
30+
</properties>
31+
32+
<!--
33+
The parent pom defines common style checks and testing strategies for our samples.
34+
Removing or replacing it should not affect the execution of the samples in anyway.
35+
-->
36+
<parent>
37+
<groupId>com.google.cloud.samples</groupId>
38+
<artifactId>shared-configuration</artifactId>
39+
<version>1.2.0</version>
40+
</parent>
41+
42+
<dependencyManagement>
43+
<dependencies>
44+
<dependency>
45+
<artifactId>libraries-bom</artifactId>
46+
<groupId>com.google.cloud</groupId>
47+
<scope>import</scope>
48+
<type>pom</type>
49+
<version>26.40.0</version>
50+
</dependency>
51+
</dependencies>
52+
</dependencyManagement>
53+
<dependencies>
54+
<dependency>
55+
<groupId>org.apache.beam</groupId>
56+
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
57+
<version>${apache_beam.version}</version>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.apache.beam</groupId>
61+
<artifactId>beam-runners-direct-java</artifactId>
62+
<version>${apache_beam.version}</version>
63+
</dependency>
64+
<dependency>
65+
<groupId>com.google.cloud.bigtable</groupId>
66+
<artifactId>bigtable-hbase-beam</artifactId>
67+
<version>2.14.0</version>
68+
</dependency>
69+
70+
<dependency>
71+
<groupId>com.google.cloud</groupId>
72+
<artifactId>google-cloud-bigtable</artifactId>
73+
</dependency>
74+
75+
<dependency>
76+
<groupId>com.google.cloud.bigtable</groupId>
77+
<artifactId>bigtable-client-core</artifactId>
78+
<version>1.29.2</version>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>junit</groupId>
83+
<artifactId>junit</artifactId>
84+
<version>4.13.2</version>
85+
<scope>test</scope>
86+
</dependency>
87+
<dependency>
88+
<groupId>com.google.truth</groupId>
89+
<artifactId>truth</artifactId>
90+
<version>1.1.5</version>
91+
<scope>test</scope>
92+
</dependency>
93+
94+
95+
</dependencies>
96+
97+
<repositories>
98+
<repository>
99+
<id>artifact-registry</id>
100+
<url>artifactregistry://us-maven.pkg.dev/cloud-bigtable-ecosystem/debug-applovin</url>
101+
<releases>
102+
<enabled>true</enabled>
103+
</releases>
104+
<snapshots>
105+
<enabled>true</enabled>
106+
</snapshots>
107+
</repository>
108+
</repositories>
109+
110+
<build>
111+
<extensions>
112+
<extension>
113+
<groupId>com.google.cloud.artifactregistry</groupId>
114+
<artifactId>artifactregistry-maven-wagon</artifactId>
115+
<version>2.2.0</version>
116+
</extension>
117+
</extensions>
118+
</build>
119+
120+
</project>

0 commit comments

Comments
 (0)