Skip to content

Commit c8081d1

Browse files
nirupa-kumarAce Nassri
nirupa-kumar
authored and
Ace Nassri
committed
Translate Automl samples (#131)
* initial translation commit * readme fix - check cloud link * first round style fixes * style fix 2 * Thanks Ace! * template path fix * Fix ENV variable for project Id (GCLOUD_PROJECT) * Translate AutoML samples * fixing lint issues * re-trigger tests
1 parent 95b2dbb commit c8081d1

File tree

8 files changed

+1136
-0
lines changed

8 files changed

+1136
-0
lines changed

‎translate/automl/automlTranslationDataset.js‎

Lines changed: 369 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,369 @@
1+
/**
2+
* Copyright 2018, Google, LLC.
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
/**
17+
* This application demonstrates how to perform basic operations on dataset
18+
* with the Google AutoML Translation API.
19+
*
20+
* For more information, see the documentation at
21+
* https://cloud.google.com/translate/automl/docs
22+
*/
23+
24+
`use strict`;
25+
26+
function createDataset(projectId, computeRegion, datasetName, source, target) {
27+
// [START automl_translation_create_dataset]
28+
const automl = require(`@google-cloud/automl`).v1beta1;
29+
30+
const client = new automl.AutoMlClient();
31+
32+
/**
33+
* TODO(developer): Uncomment the following line before running the sample.
34+
*/
35+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
36+
// const computeRegion = `region-name, e.g. "us-central1"`;
37+
// const datasetName = `name of the dataset to create, e.g. “myDataset”`;
38+
// const source = `source language code, e.g. "en" `;
39+
// const target = `target language code, e.g. "ja" `;
40+
41+
// A resource that represents Google Cloud Platform location.
42+
const projectLocation = client.locationPath(projectId, computeRegion);
43+
44+
// Specify the source and target language.
45+
const datasetSpec = {
46+
sourceLanguageCode: source,
47+
targetLanguageCode: target,
48+
};
49+
50+
// Set dataset name and dataset specification.
51+
const dataset = {
52+
displayName: datasetName,
53+
translationDatasetMetadata: datasetSpec,
54+
};
55+
56+
// Create a dataset with the dataset specification in the region.
57+
client
58+
.createDataset({parent: projectLocation, dataset: dataset})
59+
.then(responses => {
60+
const dataset = responses[0];
61+
62+
// Display the dataset information
63+
console.log(`Dataset name: ${dataset.name}`);
64+
console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`);
65+
console.log(`Dataset display name: ${dataset.displayName}`);
66+
console.log(`Dataset example count: ${dataset.exampleCount}`);
67+
console.log(`Translation dataset specification:`);
68+
console.log(
69+
`\tSource language code: ${
70+
dataset.translationDatasetMetadata.sourceLanguageCode
71+
}`
72+
);
73+
console.log(
74+
`\tTarget language code: ${
75+
dataset.translationDatasetMetadata.targetLanguageCode
76+
}`
77+
);
78+
console.log(`Dataset create time:`);
79+
console.log(`\tseconds: ${dataset.createTime.seconds}`);
80+
console.log(`\tnanos: ${dataset.createTime.nanos}`);
81+
})
82+
.catch(err => {
83+
console.error(err);
84+
});
85+
// [END automl_translation_create_dataset]
86+
}
87+
88+
function listDatasets(projectId, computeRegion, filter) {
89+
// [START automl_translation_list_datasets]
90+
const automl = require(`@google-cloud/automl`);
91+
92+
const client = new automl.v1beta1.AutoMlClient();
93+
94+
/**
95+
* TODO(developer): Uncomment the following line before running the sample.
96+
*/
97+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
98+
// const computeRegion = `region-name, e.g. "us-central1"`;
99+
// const filter = `filter expressions, must specify field e.g. “imageClassificationModelMetadata:*”`;
100+
101+
// A resource that represents Google Cloud Platform location.
102+
const projectLocation = client.locationPath(projectId, computeRegion);
103+
104+
// List all the datasets available in the region by applying filter.
105+
client
106+
.listDatasets({parent: projectLocation, filter: filter})
107+
.then(responses => {
108+
const datasets = responses[0];
109+
110+
// Display the dataset information.
111+
console.log(`List of datasets:`);
112+
datasets.forEach(dataset => {
113+
console.log(`Dataset name: ${dataset.name}`);
114+
console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`);
115+
console.log(`Dataset display name: ${dataset.displayName}`);
116+
console.log(`Dataset example count: ${dataset.exampleCount}`);
117+
console.log(`Translation dataset specification:`);
118+
console.log(
119+
`\tSource language code: ${
120+
dataset.translationDatasetMetadata.sourceLanguageCode
121+
}`
122+
);
123+
console.log(
124+
`\tTarget language code: ${
125+
dataset.translationDatasetMetadata.targetLanguageCode
126+
}`
127+
);
128+
console.log(`Dataset create time:`);
129+
console.log(`\tseconds: ${dataset.createTime.seconds}`);
130+
console.log(`\tnanos: ${dataset.createTime.nanos}`);
131+
});
132+
})
133+
.catch(err => {
134+
console.error(err);
135+
});
136+
// [START automl_translation_list_datasets]
137+
}
138+
139+
function getDataset(projectId, computeRegion, datasetId) {
140+
// [START automl_translation_get_dataset]
141+
const automl = require(`@google-cloud/automl`).v1beta1;
142+
143+
const client = new automl.AutoMlClient();
144+
145+
/**
146+
* TODO(developer): Uncomment the following line before running the sample.
147+
*/
148+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
149+
// const computeRegion = `region-name, e.g. "us-central1"`;
150+
// const datasetId = `Id of the dataset`;
151+
152+
// Get the full path of the dataset.
153+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
154+
155+
// Get complete detail of the dataset.
156+
client
157+
.getDataset({name: datasetFullId})
158+
.then(responses => {
159+
const dataset = responses[0];
160+
161+
// Display the dataset information.
162+
console.log(`Dataset name: ${dataset.name}`);
163+
console.log(`Dataset id: ${dataset.name.split(`/`).pop(-1)}`);
164+
console.log(`Dataset display name: ${dataset.displayName}`);
165+
console.log(`Dataset example count: ${dataset.exampleCount}`);
166+
console.log(`Translation dataset specification:`);
167+
console.log(
168+
`\tSource language code: ${
169+
dataset.translationDatasetMetadata.sourceLanguageCode
170+
}`
171+
);
172+
console.log(
173+
`\tTarget language code: ${
174+
dataset.translationDatasetMetadata.targetLanguageCode
175+
}`
176+
);
177+
console.log(`Dataset create time:`);
178+
console.log(`\tseconds: ${dataset.createTime.seconds}`);
179+
console.log(`\tnanos: ${dataset.createTime.nanos}`);
180+
})
181+
.catch(err => {
182+
console.error(err);
183+
});
184+
// [END automl_translation_get_dataset]
185+
}
186+
187+
function importData(projectId, computeRegion, datasetId, path) {
188+
// [START automl_translation_import_data]
189+
const automl = require(`@google-cloud/automl`).v1beta1;
190+
191+
const client = new automl.AutoMlClient();
192+
193+
/**
194+
* TODO(developer): Uncomment the following line before running the sample.
195+
*/
196+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
197+
// const computeRegion = `region-name, e.g. "us-central1"`;
198+
// const datasetId = `Id of the dataset`;
199+
// const path = `string or array of .csv paths in AutoML Vision CSV format, e.g. “gs://myproject/mytraindata.csv”;`
200+
201+
// Get the full path of the dataset.
202+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
203+
204+
// Get the multiple Google Cloud Storage URIs.
205+
const inputUris = path.split(`,`);
206+
const inputConfig = {
207+
gcsSource: {
208+
inputUris: inputUris,
209+
},
210+
};
211+
212+
// Import data from the input URI.
213+
client
214+
.importData({name: datasetFullId, inputConfig: inputConfig})
215+
.then(responses => {
216+
const operation = responses[0];
217+
console.log(`Processing import...`);
218+
return operation.promise();
219+
})
220+
.then(responses => {
221+
// The final result of the operation.
222+
if (responses[2].done === true) {
223+
console.log(`Data imported.`);
224+
}
225+
})
226+
.catch(err => {
227+
console.error(err);
228+
});
229+
// [END automl_translation_import_data]
230+
}
231+
232+
function deleteDataset(projectId, computeRegion, datasetId) {
233+
// [START automl_translation_delete_dataset]
234+
const automl = require(`@google-cloud/automl`).v1beta1;
235+
236+
const client = new automl.AutoMlClient();
237+
238+
/**
239+
* TODO(developer): Uncomment the following line before running the sample.
240+
*/
241+
// const projectId = `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`;
242+
// const computeRegion = `region-name, e.g. "us-central1"`;
243+
// const datasetId = `Id of the dataset`;
244+
245+
// Get the full path of the dataset.
246+
const datasetFullId = client.datasetPath(projectId, computeRegion, datasetId);
247+
248+
// Delete a dataset.
249+
client
250+
.deleteDataset({name: datasetFullId})
251+
.then(responses => {
252+
const operation = responses[0];
253+
return operation.promise();
254+
})
255+
.then(responses => {
256+
// The final result of the operation.
257+
if (responses[2].done === true) console.log(`Dataset deleted.`);
258+
})
259+
.catch(err => {
260+
console.error(err);
261+
});
262+
// [END automl_translation_delete_dataset]
263+
}
264+
265+
require(`yargs`)
266+
.demand(1)
267+
.options({
268+
computeRegion: {
269+
alias: `c`,
270+
type: `string`,
271+
default: process.env.REGION_NAME,
272+
requiresArg: true,
273+
description: `region name e.g. "us-central1"`,
274+
},
275+
datasetName: {
276+
alias: `n`,
277+
type: `string`,
278+
default: `testDataSet`,
279+
requiresArg: true,
280+
description: `Name of the Dataset`,
281+
},
282+
datasetId: {
283+
alias: `i`,
284+
type: `string`,
285+
requiresArg: true,
286+
description: `Id of the dataset`,
287+
},
288+
filter: {
289+
alias: `f`,
290+
default: `translationDatasetMetadata:*`,
291+
type: `string`,
292+
requiresArg: true,
293+
description: `Name of the Dataset to search for`,
294+
},
295+
multilabel: {
296+
alias: `m`,
297+
type: `string`,
298+
default: false,
299+
requiresArg: true,
300+
description:
301+
`Type of the classification problem, ` +
302+
`False - MULTICLASS, True - MULTILABEL.`,
303+
},
304+
outputUri: {
305+
alias: `o`,
306+
type: `string`,
307+
requiresArg: true,
308+
description: `URI (or local path) to export dataset`,
309+
},
310+
path: {
311+
alias: `p`,
312+
type: `string`,
313+
global: true,
314+
default: `gs://nodejs-docs-samples-vcm/en-ja.csv`,
315+
requiresArg: true,
316+
description: `URI or local path to input .csv, or array of .csv paths`,
317+
},
318+
projectId: {
319+
alias: `z`,
320+
type: `number`,
321+
default: process.env.GCLOUD_PROJECT,
322+
requiresArg: true,
323+
description: `The GCLOUD_PROJECT string, e.g. "my-gcloud-project"`,
324+
},
325+
source: {
326+
alias: `s`,
327+
type: `string`,
328+
requiresArg: true,
329+
description: `The source language to be translated from`,
330+
},
331+
target: {
332+
alias: `t`,
333+
type: `string`,
334+
requiresArg: true,
335+
description: `The target language to be translated to`,
336+
},
337+
})
338+
.command(`createDataset`, `creates a new Dataset`, {}, opts =>
339+
createDataset(
340+
opts.projectId,
341+
opts.computeRegion,
342+
opts.datasetName,
343+
opts.source,
344+
opts.target
345+
)
346+
)
347+
.command(`list-datasets`, `list all Datasets`, {}, opts =>
348+
listDatasets(opts.projectId, opts.computeRegion, opts.filter)
349+
)
350+
.command(`get-dataset`, `Get a Dataset`, {}, opts =>
351+
getDataset(opts.projectId, opts.computeRegion, opts.datasetId)
352+
)
353+
.command(`delete-dataset`, `Delete a dataset`, {}, opts =>
354+
deleteDataset(opts.projectId, opts.computeRegion, opts.datasetId)
355+
)
356+
.command(`import-data`, `Import labeled items into dataset`, {}, opts =>
357+
importData(opts.projectId, opts.computeRegion, opts.datasetId, opts.path)
358+
)
359+
.example(`node $0 create-dataset -n "newDataSet" -s "en" -t "ja"`)
360+
.example(`node $0 list-datasets -f "translationDatasetMetadata:*"`)
361+
.example(`node $0 get-dataset -i "DATASETID"`)
362+
.example(`node $0 delete-dataset -i "DATASETID"`)
363+
.example(
364+
`node $0 import-data -i "dataSetId" -p "gs://myproject/mytraindata.csv"`
365+
)
366+
.wrap(120)
367+
.recommendCommands()
368+
.help()
369+
.strict().argv;

0 commit comments

Comments
 (0)