Skip to content

Commit

Permalink
Made the test green.
Browse files Browse the repository at this point in the history
  • Loading branch information
Takashi Matsuo committed May 8, 2015
1 parent a610842 commit ab867fb
Show file tree
Hide file tree
Showing 23 changed files with 334 additions and 273 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@
.tox
coverage.xml
nosetests.xml
python-docs-samples.json
Empty file added bigquery/__init__.py
Empty file.
13 changes: 0 additions & 13 deletions bigquery/samples/__init__.py
Original file line number Diff line number Diff line change
@@ -1,13 +0,0 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
64 changes: 31 additions & 33 deletions bigquery/samples/async_query.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,24 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function # For python 2/3 interoperability
from samples.utils import get_service, paging, poll_job
import uuid

import json
import uuid

from bigquery.samples.utils import get_service
from bigquery.samples.utils import paging
from bigquery.samples.utils import poll_job


# [START async_query]
Expand All @@ -23,19 +27,19 @@ def async_query(service, project_id, query, batch=False, num_retries=5):
# don't accidentally duplicate query
job_data = {
'jobReference': {
'projectId': project_id,
'job_id': str(uuid.uuid4())
},
'projectId': project_id,
'job_id': str(uuid.uuid4())
},
'configuration': {
'query': {
'query': query,
'priority': 'BATCH' if batch else 'INTERACTIVE',
},
}
'query': {
'query': query,
'priority': 'BATCH' if batch else 'INTERACTIVE'
}
}
}
return service.jobs().insert(
projectId=project_id,
body=job_data).execute(num_retries=num_retries)
projectId=project_id,
body=job_data).execute(num_retries=num_retries)
# [END async_query]


Expand All @@ -55,7 +59,6 @@ def run(project_id, query_string, batch, num_retries, interval):
interval,
num_retries)


for page in paging(service,
service.jobs().getQueryResults,
num_retries=num_retries,
Expand All @@ -69,18 +72,13 @@ def run(project_id, query_string, batch, num_retries, interval):
def main():
project_id = raw_input("Enter the project ID: ")
query_string = raw_input("Enter the Bigquery SQL Query: ")
batch = raw_input("Run query as batch (y/n)?: ") in ('True',
'true',
'y',
'Y',
'yes',
'Yes')

batch = raw_input("Run query as batch (y/n)?: ") in (
'True', 'true', 'y', 'Y', 'yes', 'Yes')

num_retries = raw_input(
"Enter number of times to retry in case of 500 error: ")
"Enter number of times to retry in case of 500 error: ")
interval = raw_input(
"Enter how often to poll the query for completion (seconds): ")
"Enter how often to poll the query for completion (seconds): ")

for result in run(project_id, query_string, batch, num_retries, interval):
print(result)
Expand Down
52 changes: 29 additions & 23 deletions bigquery/samples/discovery_doc.py
Original file line number Diff line number Diff line change
@@ -1,35 +1,38 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
"""
A module that takes care of caching and updating discovery docs for
google-api-python-clients (until such a feature is integrated).
"""

import json
import httplib2
import os
import time

# [START build_and_update]
import httplib2

RESOURCE_PATH='..' #look for discovery docs in the parent folder
MAX_AGE = 86400 #update discovery docs older than a day
# [START build_and_update]

# A module that takes care of caching and updating discovery docs
# for google-api-python-clients (until such a feature is integrated)
RESOURCE_PATH = '..' # look for discovery docs in the parent folder
MAX_AGE = 86400 # update discovery docs older than a day
BIGQUERY_SCOPES = ['https://www.googleapis.com/auth/bigquery']


def build_and_update(api, version):
from oauth2client.client import GoogleCredentials
from googleapiclient.discovery import build_from_document


path = os.path.join(RESOURCE_PATH, '{}.{}'.format(api, version))
try:
age = time.time() - os.path.getmtime(path)
Expand All @@ -38,11 +41,14 @@ def build_and_update(api, version):
except os.error:
_update_discovery_doc(api, version, path)

credentials = GoogleCredentials.get_application_default()
if credentials.create_scoped_required():
credentials = credentials.create_scoped(BIGQUERY_SCOPES)
with open(path, 'r') as discovery_doc:
return build_from_document(discovery_doc.read(),
http=httplib2.Http(),
credentials=GoogleCredentials
.get_application_default())
http=httplib2.Http(),
credentials=credentials)


def _update_discovery_doc(api, version, path):
from apiclient.discovery import DISCOVERY_URI
Expand All @@ -61,5 +67,5 @@ def _update_discovery_doc(api, version, path):
json.dump(discovery_json, discovery_doc)
except ValueError:
raise InvalidJsonError(
'Bad JSON: %s from %s.' % (content, requested_url))
'Bad JSON: %s from %s.' % (content, requested_url))
# [END build_and_update]
49 changes: 32 additions & 17 deletions bigquery/samples/export_data_to_cloud_storage.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
from samples.utils import get_service, poll_job
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import uuid

from bigquery.samples.utils import get_service
from bigquery.samples.utils import poll_job


# [START export_table]
def export_table(service, cloud_storage_path,
Expand All @@ -9,21 +24,21 @@ def export_table(service, cloud_storage_path,
# Generate a unique job_id so retries
# don't accidentally duplicate export
job_data = {
'jobReference': {
'jobReference': {
'projectId': projectId,
'jobId': str(uuid.uuid4())
},
'configuration': {
'extract': {
'sourceTable': {
'projectId': projectId,
'jobId': str(uuid.uuid4())
},
'configuration': {
'extract': {
'sourceTable': {
'projectId': projectId,
'datasetId': datasetId,
'tableId': tableId,
},
'destinationUris': [cloud_storage_path],
}
}
'datasetId': datasetId,
'tableId': tableId,
},
'destinationUris': [cloud_storage_path],
}
}
}
return service.jobs().insert(
projectId=projectId,
body=job_data).execute(num_retries=num_retries)
Expand Down Expand Up @@ -52,11 +67,11 @@ def main():
datasetId = raw_input("Enter a dataset ID: ")
tableId = raw_input("Enter a table name to copy: ")
cloud_storage_path = raw_input(
"Enter a Google Cloud Storage URI: ")
"Enter a Google Cloud Storage URI: ")
interval = raw_input(
"Enter how often to poll the job (in seconds): ")
"Enter how often to poll the job (in seconds): ")
num_retries = raw_input(
"Enter the number of retries in case of 500 error: ")
"Enter the number of retries in case of 500 error: ")

run(cloud_storage_path,
projectId, datasetId, tableId,
Expand Down
29 changes: 16 additions & 13 deletions bigquery/samples/load_data_by_post.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# Copyright 2015, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import json

from bigquery.samples.utils import get_service, poll_job

import httplib2
from samples.utils import get_service, poll_job

from oauth2client.client import GoogleCredentials


Expand Down Expand Up @@ -64,7 +67,7 @@ def main():
datasetId = raw_input('Enter a dataset ID: ')
tableId = raw_input('Enter a table name to load the data to: ')
schema_path = raw_input(
'Enter the path to the schema file for the table: ')
'Enter the path to the schema file for the table: ')

with open(schema_path, 'r') as schema_file:
schema = schema_file.read()
Expand Down
Loading

0 comments on commit ab867fb

Please sign in to comment.