Skip to content

Use test endpoint for datalabeling integration tests. #1803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Dec 17, 2019
Merged

Use test endpoint for datalabeling integration tests. #1803

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1db4e7a
Use test endpoint for datalabeling integration tests.
dzlier-gcp Dec 11, 2019
3609f8f
Fix formatting
averikitsch Dec 13, 2019
e3438a2
Fix typo
averikitsch Dec 13, 2019
d1685f4
Fix formatting
averikitsch Dec 13, 2019
1995f79
Add quotes to variable
averikitsch Dec 13, 2019
5a3ce71
Add execeptions to method signitures.
kurtisvg Dec 17, 2019
97bfe31
Merge branch 'master' into datalabeling
dzlier-gcp Dec 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .kokoro/tests/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,11 @@ if [[ "$SCRIPT_DEBUG" != "true" ]]; then
# Setup required env variables
export GOOGLE_CLOUD_PROJECT=java-docs-samples-testing
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-acct.json
# For Tasks samples
export QUEUE_ID=my-appengine-queue
export LOCATION_ID=us-east1
# For Datalabeling samples to hit the testing endpoint
export DATALABELING_ENDPOINT="test-datalabeling.sandbox.googleapis.com:443"
source "${KOKORO_GFILE_DIR}/aws-secrets.sh"
source "${KOKORO_GFILE_DIR}/storage-hmac-credentials.sh"
source "${KOKORO_GFILE_DIR}/dlp_secrets.txt"
Expand Down
47 changes: 32 additions & 15 deletions ...ing/beta/cloud-client/src/main/java/com/example/datalabeling/CreateAnnotationSpecSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.cloud.datalabeling.v1beta1.AnnotationSpecSet;
import com.google.cloud.datalabeling.v1beta1.CreateAnnotationSpecSetRequest;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.ProjectName;
import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -32,36 +33,52 @@
class CreateAnnotationSpecSet {

// Create an annotation spec set.
static void createAnnotationSpecSet(String projectId) {
static void createAnnotationSpecSet(String projectId) throws IOException {
// String projectId = "YOUR_PROJECT_ID";

Map<String, String> annotationLabels = new HashMap<>();
annotationLabels.put("label_1", "label_1_description");
annotationLabels.put("label_2", "label_2_description");

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_create_annotation_spec_set_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_create_annotation_spec_set_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_create_annotation_spec_set_beta]
.setEndpoint(endpoint)
// [START datalabeling_create_annotation_spec_set_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add comments to each file, that this is unnecessary for users to do or something to that effect.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1. We should also add a comment why this is necessary for these snippets, and try to hide this portion from being staged in the documentation.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: It would be nice to still add a comment here (doesn't need to be in the staged version) to explain the extra code.

DataLabelingServiceClient.create(settings)) {
ProjectName projectName = ProjectName.of(projectId);

List<AnnotationSpec> annotationSpecs = new ArrayList<>();
for (Entry<String, String> entry : annotationLabels.entrySet()) {
AnnotationSpec annotationSpec = AnnotationSpec.newBuilder()
.setDisplayName(entry.getKey())
.setDescription(entry.getValue())
.build();
AnnotationSpec annotationSpec =
AnnotationSpec.newBuilder()
.setDisplayName(entry.getKey())
.setDescription(entry.getValue())
.build();
annotationSpecs.add(annotationSpec);
}

AnnotationSpecSet annotationSpecSet = AnnotationSpecSet.newBuilder()
.setDisplayName("YOUR_ANNOTATION_SPEC_SET_DISPLAY_NAME")
.setDescription("YOUR_DESCRIPTION")
.addAllAnnotationSpecs(annotationSpecs)
.build();
AnnotationSpecSet annotationSpecSet =
AnnotationSpecSet.newBuilder()
.setDisplayName("YOUR_ANNOTATION_SPEC_SET_DISPLAY_NAME")
.setDescription("YOUR_DESCRIPTION")
.addAllAnnotationSpecs(annotationSpecs)
.build();

CreateAnnotationSpecSetRequest request = CreateAnnotationSpecSetRequest.newBuilder()
.setAnnotationSpecSet(annotationSpecSet)
.setParent(projectName.toString())
.build();
CreateAnnotationSpecSetRequest request =
CreateAnnotationSpecSetRequest.newBuilder()
.setAnnotationSpecSet(annotationSpecSet)
.setParent(projectName.toString())
.build();

AnnotationSpecSet result = dataLabelingServiceClient.createAnnotationSpecSet(request);

Expand Down
37 changes: 27 additions & 10 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/CreateDataset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,29 +19,46 @@
// [START datalabeling_create_dataset_beta]
import com.google.cloud.datalabeling.v1beta1.CreateDatasetRequest;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.Dataset;
import com.google.cloud.datalabeling.v1beta1.ProjectName;
import java.io.IOException;

class CreateDataset {

// Create a dataset that is initially empty.
static void createDataset(String projectId, String datasetName) {
static void createDataset(String projectId, String datasetName) throws IOException {
// String projectId = "YOUR_PROJECT_ID";
// String datasetName = "YOUR_DATASET_DISPLAY_NAME";

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_create_dataset_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_create_dataset_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_create_dataset_beta]
.setEndpoint(endpoint)
// [START datalabeling_create_dataset_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
ProjectName projectName = ProjectName.of(projectId);

Dataset dataset = Dataset.newBuilder()
.setDisplayName(datasetName)
.setDescription("YOUR_DESCRIPTION")
.build();
Dataset dataset =
Dataset.newBuilder()
.setDisplayName(datasetName)
.setDescription("YOUR_DESCRIPTION")
.build();

CreateDatasetRequest createDatasetRequest = CreateDatasetRequest.newBuilder()
.setParent(projectName.toString())
.setDataset(dataset)
.build();
CreateDatasetRequest createDatasetRequest =
CreateDatasetRequest.newBuilder()
.setParent(projectName.toString())
.setDataset(dataset)
.build();

Dataset createdDataset = dataLabelingServiceClient.createDataset(createDatasetRequest);

Expand Down
18 changes: 16 additions & 2 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/CreateInstruction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.cloud.datalabeling.v1beta1.CreateInstructionMetadata;
import com.google.cloud.datalabeling.v1beta1.CreateInstructionRequest;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.DataType;
import com.google.cloud.datalabeling.v1beta1.Instruction;
import com.google.cloud.datalabeling.v1beta1.PdfInstruction;
Expand All @@ -31,12 +32,25 @@
class CreateInstruction {

// Create a instruction for a dataset.
static void createInstruction(String projectId, String pdfUri) {
static void createInstruction(String projectId, String pdfUri) throws IOException {
// String projectId = "YOUR_PROJECT_ID";
// String pdfUri = "gs://YOUR_BUCKET_ID/path_to_pdf_or_csv";

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_create_instruction_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_create_instruction_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_create_instruction_beta]
.setEndpoint(endpoint)
// [START datalabeling_create_instruction_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
ProjectName projectName = ProjectName.of(projectId);

// There are two types of instructions: CSV (CsvInstruction) or PDF (PdfInstruction)
Expand Down
22 changes: 19 additions & 3 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/ExportData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
// [START datalabeling_export_data_beta]
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.ExportDataOperationMetadata;
import com.google.cloud.datalabeling.v1beta1.ExportDataOperationResponse;
import com.google.cloud.datalabeling.v1beta1.ExportDataRequest;
Expand All @@ -33,7 +34,8 @@
class ExportData {

// Export data from an annotated dataset.
static void exportData(String datasetName, String annotatedDatasetName, String gcsOutputUri) {
static void exportData(String datasetName, String annotatedDatasetName, String gcsOutputUri)
throws IOException {
// String datasetName = DataLabelingServiceClient.formatDatasetName(
// "YOUR_PROJECT_ID", "YOUR_DATASETS_UUID");
// String annotatedDatasetName = DataLabelingServiceClient.formatAnnotatedDatasetName(
Expand All @@ -42,7 +44,21 @@ static void exportData(String datasetName, String annotatedDatasetName, String g
// "YOUR_ANNOTATED_DATASET_UUID");
// String gcsOutputUri = "gs://YOUR_BUCKET_ID/export_path";

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_export_data_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_export_data_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_export_data_beta]
.setEndpoint(endpoint)
// [START datalabeling_export_data_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
GcsDestination gcsDestination = GcsDestination.newBuilder()
.setOutputUri(gcsOutputUri)
.setMimeType("text/csv")
Expand Down Expand Up @@ -75,4 +91,4 @@ static void exportData(String datasetName, String annotatedDatasetName, String g
}
}
}
// [END datalabeling_export_data_beta]
// [END datalabeling_export_data_beta]
19 changes: 17 additions & 2 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/ImportData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
// [START datalabeling_import_data_beta]
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.DataType;
import com.google.cloud.datalabeling.v1beta1.GcsSource;
import com.google.cloud.datalabeling.v1beta1.ImportDataOperationMetadata;
Expand All @@ -31,12 +32,26 @@
class ImportData {

// Import data to an existing dataset.
static void importData(String datasetName, String gcsSourceUri) {
static void importData(String datasetName, String gcsSourceUri) throws IOException {
// String datasetName = DataLabelingServiceClient.formatDatasetName(
// "YOUR_PROJECT_ID", "YOUR_DATASETS_UUID");
// String gcsSourceUri = "gs://YOUR_BUCKET_ID/path_to_data";

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_import_data_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_import_data_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_import_data_beta]
.setEndpoint(endpoint)
// [START datalabeling_import_data_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
GcsSource gcsSource = GcsSource.newBuilder()
.setInputUri(gcsSourceUri)
.setMimeType("text/csv")
Expand Down
18 changes: 16 additions & 2 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/LabelImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.datalabeling.v1beta1.AnnotatedDataset;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.HumanAnnotationConfig;
import com.google.cloud.datalabeling.v1beta1.ImageClassificationConfig;
import com.google.cloud.datalabeling.v1beta1.LabelImageRequest;
Expand All @@ -35,7 +36,7 @@ class LabelImage {
static void labelImage(
String formattedInstructionName,
String formattedAnnotationSpecSetName,
String formattedDatasetName) {
String formattedDatasetName) throws IOException {
// String formattedInstructionName = DataLabelingServiceClient.formatInstructionName(
// "YOUR_PROJECT_ID", "YOUR_INSTRUCTION_UUID");
// String formattedAnnotationSpecSetName =
Expand All @@ -44,8 +45,21 @@ static void labelImage(
// String formattedDatasetName = DataLabelingServiceClient.formatDatasetName(
// "YOUR_PROJECT_ID", "YOUR_DATASET_UUID");

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_label_image_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_label_image_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_label_image_beta]
.setEndpoint(endpoint)
// [START datalabeling_label_image_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
HumanAnnotationConfig humanAnnotationConfig =
HumanAnnotationConfig.newBuilder()
.setAnnotatedDatasetDisplayName("annotated_displayname")
Expand Down
18 changes: 16 additions & 2 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/LabelText.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.datalabeling.v1beta1.AnnotatedDataset;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.HumanAnnotationConfig;
import com.google.cloud.datalabeling.v1beta1.LabelOperationMetadata;
import com.google.cloud.datalabeling.v1beta1.LabelTextRequest;
Expand All @@ -35,7 +36,7 @@ class LabelText {
static void labelText(
String formattedInstructionName,
String formattedAnnotationSpecSetName,
String formattedDatasetName) {
String formattedDatasetName) throws IOException {
// String formattedInstructionName = DataLabelingServiceClient.formatInstructionName(
// "YOUR_PROJECT_ID", "YOUR_INSTRUCTION_UUID");
// String formattedAnnotationSpecSetName =
Expand All @@ -44,8 +45,21 @@ static void labelText(
// String formattedDatasetName = DataLabelingServiceClient.formatDatasetName(
// "YOUR_PROJECT_ID", "YOUR_DATASET_UUID");

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_label_text_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_label_text_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_label_text_beta]
.setEndpoint(endpoint)
// [START datalabeling_label_text_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
HumanAnnotationConfig humanAnnotationConfig =
HumanAnnotationConfig.newBuilder()
.setAnnotatedDatasetDisplayName("annotated_displayname")
Expand Down
20 changes: 17 additions & 3 deletions datalabeling/beta/cloud-client/src/main/java/com/example/datalabeling/LabelVideo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.datalabeling.v1beta1.AnnotatedDataset;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.HumanAnnotationConfig;
import com.google.cloud.datalabeling.v1beta1.LabelOperationMetadata;
import com.google.cloud.datalabeling.v1beta1.LabelVideoRequest;
Expand All @@ -33,7 +34,7 @@ class LabelVideo {

// Start a Video Labeling Task
static void labelVideo(String formattedInstructionName, String formattedAnnotationSpecSetName,
String formattedDatasetName) {
String formattedDatasetName) throws IOException {
// String formattedInstructionName = DataLabelingServiceClient.formatInstructionName(
// "YOUR_PROJECT_ID", "YOUR_INSTRUCTION_UUID");
// String formattedAnnotationSpecSetName =
Expand All @@ -42,8 +43,21 @@ static void labelVideo(String formattedInstructionName, String formattedAnnotati
// String formattedDatasetName = DataLabelingServiceClient.formatDatasetName(
// "YOUR_PROJECT_ID", "YOUR_DATASET_UUID");

try (DataLabelingServiceClient dataLabelingServiceClient = DataLabelingServiceClient.create()) {
// [END datalabeling_label_video_beta]
String endpoint = System.getenv("DATALABELING_ENDPOINT");
if (endpoint == null) {
endpoint = DataLabelingServiceSettings.getDefaultEndpoint();
}
// [START datalabeling_label_video_beta]

DataLabelingServiceSettings settings = DataLabelingServiceSettings
.newBuilder()
// [END datalabeling_label_video_beta]
.setEndpoint(endpoint)
// [START datalabeling_label_video_beta]
.build();
try (DataLabelingServiceClient dataLabelingServiceClient =
DataLabelingServiceClient.create(settings)) {
HumanAnnotationConfig humanAnnotationConfig = HumanAnnotationConfig.newBuilder()
.setAnnotatedDatasetDisplayName("annotated_displayname")
.setAnnotatedDatasetDescription("annotated_description")
Expand Down Expand Up @@ -80,4 +94,4 @@ static void labelVideo(String formattedInstructionName, String formattedAnnotati
}
}
}
// [END datalabeling_label_video_beta]
// [END datalabeling_label_video_beta]
Loading