Skip to content

Commit 8f69183

Browse files
jmdobryjmdobry
committed
Merge pull request #91 from GoogleCloudPlatform/bigquery
BigQuery samples.
2 parents a6232bb + 1557dc7 commit 8f69183

17 files changed

+769
-3
lines changed

‎README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ This repository holds Node.js samples used throughout [cloud.google.com]().
88
## Table of Contents
99

1010
* [Google App Engine](#google-app-engine)
11+
* [Google BigQuery](#google-bigquery)
1112
* [Google Cloud Functions](#google-cloud-functions)
1213
* [Google Cloud Logging](#google-cloud-logging)
1314
* [Google Cloud Pub/Sub](#google-cloud-pubsub)
@@ -69,6 +70,13 @@ __Other Examples__
6970
- Reading/writing from/to disk - [Source code][aedisk_1]
7071
- Serving static files - [Source code][aestaticfiles_1]
7172

73+
## Google BigQuery
74+
75+
- Query sample - [Source code][bigquery_query_1] | [Documentation][bigquery_query_2]
76+
- Dataset size sample - [Source code][bigquery_size_1]
77+
- Load from file sample - [Source code][bigquery_file_1] | [Documentation][bigquery_file_2]
78+
- Load from GCS sample - [Source code][bigquery_gcs_1] | [Documentation][bigquery_gcs_2]
79+
7280
## Google Cloud Datastore
7381

7482
- Tasks sample - [Source code][datastore_1] | [Documentation][datastore_2]
@@ -124,6 +132,8 @@ See [CONTRIBUTING.md](https://github.com/GoogleCloudPlatform/nodejs-docs-samples
124132
1. Start Redis
125133
1. Start Memcached
126134
1. Set the `GCLOUD_PROJECT` environment variable to id of your project
135+
1. Set the `TEST_BUCKET_NAME` environment variable to the name of a test Google
136+
Cloud Storage bucket.
127137
1. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to
128138
a service account file. You can download one from your Google project's
129139
"permissions" page.
@@ -253,6 +263,17 @@ See [LICENSE](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/ma
253263
[aeextending_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/appengine/extending-runtime
254264
[aestaticfiles_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/appengine/static-files
255265

266+
[bigquery_query_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/getting_started.js
267+
[bigquery_query_2]: https://cloud.google.com/datastore/docs/concepts/overview
268+
269+
[bigquery_size_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/dataset_size.js
270+
271+
[bigquery_file_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/load_data_from_csv.js
272+
[bigquery_file_2]: https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatapostrequest
273+
274+
[bigquery_gcs_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/bigquery/load_data_from_gcs.js
275+
[bigquery_gcs_2]: https://cloud.google.com/bigquery/loading-data-into-bigquery#loaddatagcs
276+
256277
[datastore_1]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/blob/master/datastore/tasks.js
257278
[datastore_2]: https://cloud.google.com/datastore/docs/concepts/overview
258279

‎bigquery/README.md

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
## BigQuery Samples
2+
3+
These samples require two environment variables to be set:
4+
5+
- `GOOGLE_APPLICATION_CREDENTIALS` - Path to a service account file. You can
6+
download one from your Google project's "permissions" page.
7+
- `GCLOUD_PROJECT` - Id of your Google project.
8+
9+
## Run the samples
10+
11+
Install dependencies:
12+
13+
npm install
14+
15+
### getting_started.js
16+
17+
npm run getting_started
18+
19+
### dataset_size.js
20+
21+
Usage: `npm run dataset_size -- <projectId> <datasetId>`
22+
23+
Example:
24+
25+
npm run dataset_size -- bigquery-public-data hacker_news
26+
27+
### load_data_from_csv.js
28+
29+
Usage: `npm run load_data_from_csv -- <path-to-file> <dataset-id> <table-name>`
30+
31+
Example:
32+
33+
npm run load_data_from_csv -- data.csv my-dataset my-table
34+
35+
### load_data_from_gcs.js
36+
37+
Usage: `npm run load_data_from_gcs -- <bucket-name> <filename> <dataset-id> <table-name>`
38+
39+
Example:
40+
41+
npm run load_data_from_gcs -- my-bucket data.csv my-dataset my-table

‎bigquery/dataset_size.js

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
// Copyright 2016, Google, Inc.
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
'use strict';
15+
16+
var async = require('async');
17+
18+
// [START auth]
19+
// You must set the GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
20+
// environment variables to run this sample
21+
var projectId = process.env.GCLOUD_PROJECT;
22+
23+
// Initialize gcloud
24+
var gcloud = require('gcloud')({
25+
projectId: projectId
26+
});
27+
28+
// Get a reference to the bigquery component
29+
var bigquery = gcloud.bigquery();
30+
// [END auth]
31+
32+
// not going to use this bigquery instance
33+
bigquery = undefined;
34+
35+
// [START list_tables]
36+
/**
37+
* Retrieve all tables for the specified dataset.
38+
*
39+
* @param {Object} bigquery gcloud-node bigquery client.
40+
* @param {string} datasetId Dataset of the tables to retrieve.
41+
* @param {string} [pageToken] Page to retrieve.
42+
* @param {Function} callback Callback function.
43+
*/
44+
function getAllTablesExample(bigquery, datasetId, pageToken, callback) {
45+
if (typeof pageToken === 'function') {
46+
callback = pageToken;
47+
pageToken = undefined;
48+
}
49+
var dataset = bigquery.dataset(datasetId);
50+
var options = {};
51+
if (pageToken) {
52+
options.pageToken = pageToken;
53+
}
54+
55+
// Grab paginated tables
56+
dataset.getTables(options, function (err, tables, nextQuery) {
57+
// Quit on error
58+
if (err) {
59+
return callback(err);
60+
}
61+
62+
// There is another page of tables
63+
if (nextQuery) {
64+
// Grab the remaining pages of tables recursively
65+
return getAllTablesExample(
66+
datasetId,
67+
nextQuery.token,
68+
function (err, _tables) {
69+
if (err) {
70+
return callback(err);
71+
}
72+
callback(null, tables.concat(_tables));
73+
}
74+
);
75+
}
76+
// Last page of tables
77+
return callback(null, tables);
78+
});
79+
}
80+
// [END list_tables]
81+
82+
// [START get_size]
83+
/**
84+
* Retrieve the size of the specified dataset.
85+
*
86+
* @param {string} projectId The project, .e.g. "bigquery-public-data"
87+
* @param {string} datasetId The dataset, e.g. "hacker_news"
88+
* @param {Function} callback Callback function.
89+
*/
90+
function getSizeExample(projectId, datasetId, callback) {
91+
if (!projectId) {
92+
return callback(new Error('projectId is required!'));
93+
}
94+
if (!datasetId) {
95+
return callback(new Error('datasetId is require!'));
96+
}
97+
98+
var gcloud = require('gcloud')({
99+
projectId: projectId || process.env.GCLOUD_PROJECT
100+
});
101+
var bigquery = gcloud.bigquery();
102+
103+
// Fetch all tables in the dataset
104+
getAllTablesExample(bigquery, datasetId, function (err, tables) {
105+
return async.parallel(tables.map(function (table) {
106+
return function (cb) {
107+
// Fetch more detailed info for each table
108+
table.get(function (err, tableInfo) {
109+
if (err) {
110+
return cb(err);
111+
}
112+
// Return numBytes converted to Megabytes
113+
var numBytes = tableInfo.metadata.numBytes;
114+
return cb(null, (parseInt(numBytes, 10) / 1000) / 1000);
115+
});
116+
};
117+
}), function (err, sizes) {
118+
if (err) {
119+
return callback(err);
120+
}
121+
var sum = sizes.reduce(function (cur, prev) {
122+
return cur + prev;
123+
}, 0);
124+
return callback(null, sum);
125+
});
126+
});
127+
}
128+
// [END get_size]
129+
130+
// Run the examples
131+
exports.main = function (projectId, datasetId, cb) {
132+
getSizeExample(projectId, datasetId, function (err, sum) {
133+
if (err) {
134+
return cb(err);
135+
}
136+
var size = 'MB';
137+
if (sum > 1000) {
138+
sum = sum / 1000;
139+
size = 'GB';
140+
}
141+
if (sum > 1000) {
142+
sum = sum / 1000;
143+
size = 'TB';
144+
}
145+
cb(null, '' + sum.toPrecision(5) + ' ' + size);
146+
});
147+
};
148+
149+
if (module === require.main) {
150+
var args = process.argv.slice(2);
151+
if (args.length !== 2) {
152+
throw new Error('Usage: node dataset_size.js <projectId> <datasetId>');
153+
}
154+
exports.main(
155+
args[0],
156+
args[1],
157+
console.log
158+
);
159+
}

‎bigquery/getting_started.js

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Copyright 2016, Google, Inc.
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
// [START complete]
15+
'use strict';
16+
17+
// [START auth]
18+
// You must set the GOOGLE_APPLICATION_CREDENTIALS and GCLOUD_PROJECT
19+
// environment variables to run this sample
20+
var projectId = process.env.GCLOUD_PROJECT;
21+
22+
// Initialize gcloud
23+
var gcloud = require('gcloud')({
24+
projectId: projectId
25+
});
26+
27+
// Get a reference to the bigquery component
28+
var bigquery = gcloud.bigquery();
29+
// [END auth]
30+
31+
// [START print]
32+
function printExample(rows) {
33+
console.log('Query Results:');
34+
rows.forEach(function (row) {
35+
var str = '';
36+
for (var key in row) {
37+
if (str) {
38+
str += '\t';
39+
}
40+
str += key + ': ' + row[key];
41+
}
42+
console.log(str);
43+
});
44+
}
45+
// [END print]
46+
47+
// [START query]
48+
/**
49+
* Run an example query.
50+
*
51+
* @param {Function} callback Callback function.
52+
*/
53+
function queryExample(callback) {
54+
var query = 'SELECT TOP(corpus, 10) as title, COUNT(*) as unique_words\n' +
55+
'FROM [publicdata:samples.shakespeare];';
56+
57+
bigquery.query(query, function(err, rows) {
58+
if (err) {
59+
return callback(err);
60+
}
61+
62+
printExample(rows);
63+
callback(null, rows);
64+
});
65+
}
66+
// [END query]
67+
68+
// [END complete]
69+
70+
// Run the examples
71+
exports.main = function (cb) {
72+
queryExample(cb);
73+
};
74+
75+
if (module === require.main) {
76+
exports.main(
77+
console.log
78+
);
79+
}

0 commit comments

Comments
 (0)