From 586624e52abff938b893cc31eee6ba2822e84bc4 Mon Sep 17 00:00:00 2001 From: Gus Class Date: Tue, 26 Sep 2017 13:21:47 -0700 Subject: [PATCH] Final additions in private beta (#1136) * Final additions in private beta * Adds HTTP client and state support * Fixes rouge space * Remove invalid message on error. --- iot/api-client/http_example/README.md | 50 ++++++ .../http_example/cloudiot_http_example.py | 161 ++++++++++++++++++ iot/api-client/http_example/requirements.txt | 3 + iot/api-client/manager/manager.py | 38 ++++- iot/api-client/manager/manager_test.py | 10 ++ iot/api-client/mqtt_example/README.md | 43 +++-- .../mqtt_example/cloudiot_mqtt_example.py | 121 ++++++------- 7 files changed, 336 insertions(+), 90 deletions(-) create mode 100644 iot/api-client/http_example/README.md create mode 100644 iot/api-client/http_example/cloudiot_http_example.py create mode 100644 iot/api-client/http_example/requirements.txt diff --git a/iot/api-client/http_example/README.md b/iot/api-client/http_example/README.md new file mode 100644 index 000000000000..0197fa0d73cd --- /dev/null +++ b/iot/api-client/http_example/README.md @@ -0,0 +1,50 @@ +# Cloud IoT Core Python HTTP example + +This sample app publishes data to Cloud Pub/Sub using the HTTP bridge provided +as part of Google Cloud IoT Core. For detailed running instructions see the +[HTTP code samples +guide](https://cloud.google.com/iot/docs/protocol_bridge_guide). + +# Setup + +1. Use virtualenv to create a local Python environment. + +``` +virtualenv env && source env/bin/activate +``` + +1. Install the dependencies + +``` +pip install -r requirements.txt +``` + +# Running the Sample + +The following snippet summarizes usage: + +``` +cloudiot_http_example.py [-h] + --project_id PROJECT_ID + --registry_id REGISTRY_ID + --device_id DEVICE_ID + --private_key_file PRIVATE_KEY_FILE + --algorithm {RS256,ES256} + --message_type={event,state} + [--cloud_region CLOUD_REGION] + [--ca_certs CA_CERTS] + [--num_messages NUM_MESSAGES] +``` + +For example, if your project ID is `blue-jet-123`, the following example shows +how you would execute using the configuration from the HTTP code samples guide: + +``` +python cloudiot_http_example.py \ + --registry_id=my-registry \ + --project_id=blue-jet-123 \ + --device_id=my-python-device \ + --message_type=event \ + --algorithm=RS256 \ + --private_key_file=../rsa_private.pem +``` diff --git a/iot/api-client/http_example/cloudiot_http_example.py b/iot/api-client/http_example/cloudiot_http_example.py new file mode 100644 index 000000000000..aeceeaf005a4 --- /dev/null +++ b/iot/api-client/http_example/cloudiot_http_example.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python sample for connecting to Google Cloud IoT Core via HTTP, using JWT. +This example connects to Google Cloud IoT Core via HTTP, using a JWT for device +authentication. After connecting, by default the device publishes 100 messages +to the server at a rate of one per second, and then exits. +Before you run the sample, you must register your device as described in the +README in the parent folder. +""" + +import argparse +import base64 +import datetime +import json +import time + +import jwt +import requests + + +_BASE_URL = 'https://cloudiot-device.googleapis.com/v1beta1' + + +def create_jwt(project_id, private_key_file, algorithm): + """Creates a JWT (https://jwt.io) to authenticate this device. + Args: + project_id: The cloud project ID this device belongs to + private_key_file: A path to a file containing either an RSA256 or + ES256 private key. + algorithm: The encryption algorithm to use. Either 'RS256' or + 'ES256' + Returns: + A JWT generated from the given project_id and private key, which + expires in 20 minutes. After 20 minutes, your client will be + disconnected, and a new JWT will have to be generated. + Raises: + ValueError: If the private_key_file does not contain a known key. + """ + + token = { + # The time the token was issued. + 'iat': datetime.datetime.utcnow(), + # Token expiration time. + 'exp': datetime.datetime.utcnow() + datetime.timedelta(minutes=60), + # The audience field should always be set to the GCP project id. + 'aud': project_id + } + + # Read the private key file. + with open(private_key_file, 'r') as f: + private_key = f.read() + + print('Creating JWT using {} from private key file {}'.format( + algorithm, private_key_file)) + + return jwt.encode(token, private_key, algorithm=algorithm) + + +def parse_command_line_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser(description=( + 'Example Google Cloud IoT Core HTTP device connection code.')) + parser.add_argument( + '--project_id', required=True, help='GCP cloud project name') + parser.add_argument( + '--registry_id', required=True, help='Cloud IoT Core registry id') + parser.add_argument( + '--device_id', required=True, help='Cloud IoT Core device id') + parser.add_argument( + '--private_key_file', + required=True, + help='Path to private key file.') + parser.add_argument( + '--algorithm', + choices=('RS256', 'ES256'), + required=True, + help='The encryption algorithm to use to generate the JWT.') + parser.add_argument( + '--cloud_region', default='us-central1', help='GCP cloud region') + parser.add_argument( + '--ca_certs', + default='roots.pem', + help=('CA root from https://pki.google.com/roots.pem')) + parser.add_argument( + '--num_messages', + type=int, + default=100, + help='Number of messages to publish.') + parser.add_argument( + '--message_type', + choices=('event', 'state'), + default='event', + required=True, + help=('Indicates whether the message to be published is a ' + 'telemetry event or a device state message.')) + parser.add_argument( + '--base_url', + default=_BASE_URL, + help=('Base URL for the Cloud IoT Core Device Service API')) + + return parser.parse_args() + + +def main(): + args = parse_command_line_args() + + # Publish to the events or state topic based on the flag. + url_suffix = 'publishEvent' if args.message_type == 'event' else 'setState' + + publish_url = ( + '{}/projects/{}/locations/{}/registries/{}/devices/{}:{}').format( + args.base_url, args.project_id, args.cloud_region, + args.registry_id, args.device_id, url_suffix) + + jwt_token = create_jwt( + args.project_id, args.private_key_file, args.algorithm) + + headers = { + 'Authorization': 'Bearer {}'.format(jwt_token), + 'Content-Type': 'application/json' + } + + # Publish num_messages mesages to the HTTP bridge once per second. + for i in range(1, args.num_messages + 1): + payload = '{}/{}-payload-{}'.format( + args.registry_id, args.device_id, i) + print('Publishing message {}/{}: \'{}\''.format( + i, args.num_messages, payload)) + body = None + if args.message_type == 'event': + body = {'binary_data': base64.urlsafe_b64encode(payload)} + else: + body = { + 'state': {'binary_data': base64.urlsafe_b64encode(payload)} + } + + resp = requests.post( + publish_url, data=json.dumps(body), headers=headers) + + print('HTTP response: ', resp) + + # Send events every second. State should not be updated as often + time.sleep(1 if args.message_type == 'event' else 5) + print('Finished.') + + +if __name__ == '__main__': + main() diff --git a/iot/api-client/http_example/requirements.txt b/iot/api-client/http_example/requirements.txt new file mode 100644 index 000000000000..543a5b71ae84 --- /dev/null +++ b/iot/api-client/http_example/requirements.txt @@ -0,0 +1,3 @@ +cryptography==2.0.3 +pyjwt==1.5.3 +requests==2.18.4 diff --git a/iot/api-client/manager/manager.py b/iot/api-client/manager/manager.py index 72305f34d92b..e2e9a8e2e1a3 100644 --- a/iot/api-client/manager/manager.py +++ b/iot/api-client/manager/manager.py @@ -69,7 +69,7 @@ def get_client(service_account_json, api_key): provided API key and creating a service object using the service account credentials JSON.""" api_scopes = ['https://www.googleapis.com/auth/cloud-platform'] - api_version = 'v1beta1' + api_version = 'v1' discovery_api = 'https://cloudiot.googleapis.com/$discovery/rest' service_name = 'cloudiotcore' @@ -173,7 +173,7 @@ def delete_device( def delete_registry( - service_account_json, api_key, project_id, cloud_region, registry_id): + service_account_json, api_key, project_id, cloud_region, registry_id): """Deletes the specified registry.""" print('Delete registry') client = get_client(service_account_json, api_key) @@ -216,6 +216,23 @@ def get_device( return device +def get_state( + service_account_json, api_key, project_id, cloud_region, registry_id, + device_id): + """Retrieve a device's state blobs.""" + client = get_client(service_account_json, api_key) + registry_name = 'projects/{}/locations/{}/registries/{}'.format( + project_id, cloud_region, registry_id) + + device_name = '{}/devices/{}'.format(registry_name, device_id) + devices = client.projects().locations().registries().devices() + state = devices.states().list(name=device_name, numStates=5).execute() + + print('State: {}\n'.format(state)) + + return state + + def list_devices( service_account_json, api_key, project_id, cloud_region, registry_id): """List all devices in the registry.""" @@ -261,9 +278,9 @@ def create_registry( project_id, cloud_region) body = { - 'eventNotificationConfig': { + 'eventNotificationConfigs': [{ 'pubsubTopicName': pubsub_topic - }, + }], 'id': registry_id } request = client.projects().locations().registries().create( @@ -274,6 +291,7 @@ def create_registry( print('Created registry') return response except HttpError: + print('Error, registry not created') return "" @@ -425,7 +443,8 @@ def parse_command_line_args(): command.add_parser('delete-device', help=delete_device.__doc__) command.add_parser('delete-registry', help=delete_registry.__doc__) command.add_parser('get', help=get_device.__doc__) - command.add_parser('get-registry', help=get_device.__doc__) + command.add_parser('get-registry', help=get_registry.__doc__) + command.add_parser('get-state', help=get_state.__doc__) command.add_parser('list', help=list_devices.__doc__) command.add_parser('list-registries', help=list_registries.__doc__) command.add_parser('patch-es256', help=patch_es256_auth.__doc__) @@ -436,6 +455,10 @@ def parse_command_line_args(): def run_command(args): """Calls the program using the specified command.""" + if args.project_id is None: + print('You must specify a project ID or set the environment variable.') + return + if args.command == 'create-rsa256': create_rs256_device( args.service_account_json, args.api_key, args.project_id, @@ -476,6 +499,11 @@ def run_command(args): args.service_account_json, args.api_key, args.project_id, args.cloud_region, args.registry_id, args.device_id) + elif args.command == 'get-state': + get_state( + args.service_account_json, args.api_key, args.project_id, + args.cloud_region, args.registry_id, args.device_id) + elif args.command == 'list': list_devices( args.service_account_json, args.api_key, args.project_id, diff --git a/iot/api-client/manager/manager_test.py b/iot/api-client/manager/manager_test.py index ab7c29853c38..de155e764552 100644 --- a/iot/api-client/manager/manager_test.py +++ b/iot/api-client/manager/manager_test.py @@ -101,6 +101,10 @@ def test_add_delete_rs256_device(test_topic, capsys): service_account_json, api_key, project_id, cloud_region, registry_id, device_id) + manager.get_state( + service_account_json, api_key, project_id, cloud_region, + registry_id, device_id) + manager.delete_device( service_account_json, api_key, project_id, cloud_region, registry_id, device_id) @@ -111,6 +115,7 @@ def test_add_delete_rs256_device(test_topic, capsys): out, _ = capsys.readouterr() assert 'format : RSA_X509_PEM' in out + assert 'State: {' in out def test_add_delete_es256_device(test_topic, capsys): @@ -127,6 +132,10 @@ def test_add_delete_es256_device(test_topic, capsys): service_account_json, api_key, project_id, cloud_region, registry_id, device_id) + manager.get_state( + service_account_json, api_key, project_id, cloud_region, + registry_id, device_id) + manager.delete_device( service_account_json, api_key, project_id, cloud_region, registry_id, device_id) @@ -137,6 +146,7 @@ def test_add_delete_es256_device(test_topic, capsys): out, _ = capsys.readouterr() assert 'format : ES256_PEM' in out + assert 'State: {' in out def test_add_patch_delete_rs256(test_topic, capsys): diff --git a/iot/api-client/mqtt_example/README.md b/iot/api-client/mqtt_example/README.md index d36ce358a5b1..1d8760eb3620 100644 --- a/iot/api-client/mqtt_example/README.md +++ b/iot/api-client/mqtt_example/README.md @@ -3,40 +3,49 @@ This sample app publishes data to Cloud Pub/Sub using the MQTT bridge provided as part of Google Cloud IoT Core. -For detailed running instructions see the [MQTT code samples guide](https://cloud.google.com/iot/docs/protocol_bridge_guide). +For detailed running instructions see the [MQTT code samples +guide](https://cloud.google.com/iot/docs/protocol_bridge_guide). # Setup -1. Use virtualenv to create a local Python environment. +1. Follow the instructions in the [parent README](../README.md). + +2. Use virtualenv to create a local Python environment. virtualenv env && source env/bin/activate -2. Install the dependencies +3. Install the dependencies pip install -r requirements.txt +4. Download the CA root certificates from pki.google.com into the same + directory as the example script: + + wget https://pki.google.com/roots.pem + # Running the Sample The following snippet summarizes usage: cloudiot_mqtt_example.py [-h] - --project_id PROJECT_ID - --registry_id REGISTRY_ID - --device_id DEVICE_ID - --private_key_file PRIVATE_KEY_FILE - --algorithm {RS256,ES256} - [--cloud_region CLOUD_REGION] - [--ca_certs CA_CERTS] - [--num_messages NUM_MESSAGES] - [--mqtt_bridge_hostname MQTT_BRIDGE_HOSTNAME] - [--mqtt_bridge_port MQTT_BRIDGE_PORT] + --project_id=PROJECT_ID + --registry_id=REGISTRY_ID + --device_id=DEVICE_ID + --private_key_file=PRIVATE_KEY_FILE + --algorithm={RS256,ES256} + [--cloud_region=CLOUD_REGION] + [--ca_certs=CA_CERTS] + [--num_messages=NUM_MESSAGES] + [--mqtt_bridge_hostname=MQTT_BRIDGE_HOSTNAME] + [--mqtt_bridge_port=MQTT_BRIDGE_PORT] + [--message_type={event,state}] For example, if your project ID is `blue-jet-123`, the following example shows how you would execute using the configuration from the MQTT code samples guide: python cloudiot_mqtt_example.py \ - --registry_id my-registry \ + --registry_id=my-registry \ --project_id=blue-jet-123 \ - --device_id my-python-device \ - --algorithm RS256 \ - --private_key_file ../rsa_private.pem + --device_id=my-python-device \ + --algorithm=RS256 \ + --private_key_file=../rsa_private.pem diff --git a/iot/api-client/mqtt_example/cloudiot_mqtt_example.py b/iot/api-client/mqtt_example/cloudiot_mqtt_example.py index 31142faa6815..9498e35c2735 100644 --- a/iot/api-client/mqtt_example/cloudiot_mqtt_example.py +++ b/iot/api-client/mqtt_example/cloudiot_mqtt_example.py @@ -6,38 +6,24 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Python sample for connecting to Google Cloud IoT Core via MQTT, using JWT. - This example connects to Google Cloud IoT Core via MQTT, using a JWT for device authentication. After connecting, by default the device publishes 100 messages to the device's MQTT topic at a rate of one per second, and then exits. - -Before you run the sample, you must register your device as described in the -README in the parent folder. - -After registering the device, download Google's CA root certificates with - - wget https://pki.google.com/roots.pem - -and run this script with the corresponding algorithm flag, for example: - - python cloudiot_mqtt_example.py --project_id=my-project-id \ - --registry_id=my-registry-id \ - --device_id=my-device-id \ - --private_key_file=rsa_private.pem \ - --algorithm=RS256 +Before you run the sample, you must follow the instructions in the README +for this sample. """ import argparse import datetime +import os import time import jwt @@ -46,41 +32,34 @@ def create_jwt(project_id, private_key_file, algorithm): """Creates a JWT (https://jwt.io) to establish an MQTT connection. - - Args: - project_id: The cloud project ID this device belongs to - - private_key_file: A path to a file containing either an RSA256 or ES256 - private key. - - algorithm: The encryption algorithm to use. Either 'RS256' or 'ES256' - - Returns: - An MQTT generated from the given project_id and private key, which - expires in 20 minutes. After 20 minutes, your client will be - disconnected, and a new JWT will have to be generated. - - Raises: - ValueError: If the private_key_file does not contain a known key. - """ + Args: + project_id: The cloud project ID this device belongs to + private_key_file: A path to a file containing either an RSA256 or + ES256 private key. + algorithm: The encryption algorithm to use. Either 'RS256' or 'ES256' + Returns: + An MQTT generated from the given project_id and private key, which + expires in 20 minutes. After 20 minutes, your client will be + disconnected, and a new JWT will have to be generated. + Raises: + ValueError: If the private_key_file does not contain a known key. + """ token = { - # The time that the token was issued at - 'iat': datetime.datetime.utcnow(), - # When this token expires. The device will be disconnected after the - # token expires, and will have to reconnect. - 'exp': datetime.datetime.utcnow() + datetime.timedelta(minutes=60), - # The audience field should always be set to the GCP project id. - 'aud': project_id + # The time that the token was issued at + 'iat': datetime.datetime.utcnow(), + # The time the token expires. + 'exp': datetime.datetime.utcnow() + datetime.timedelta(minutes=60), + # The audience field should always be set to the GCP project id. + 'aud': project_id } # Read the private key file. with open(private_key_file, 'r') as f: private_key = f.read() - print( - 'Creating JWT using {} from private key file {}'.format( - algorithm, private_key_file)) + print('Creating JWT using {} from private key file {}'.format( + algorithm, private_key_file)) return jwt.encode(token, private_key, algorithm=algorithm) @@ -107,25 +86,19 @@ def on_publish(unused_client, unused_userdata, unused_mid): def parse_command_line_args(): """Parse command line arguments.""" - parser = argparse.ArgumentParser( - description=( - 'Example Google Cloud IoT Core MQTT device connection code.')) + parser = argparse.ArgumentParser(description=( + 'Example Google Cloud IoT Core MQTT device connection code.')) parser.add_argument( '--project_id', - required=True, + default=os.environ.get('GOOGLE_CLOUD_PROJECT'), help='GCP cloud project name') parser.add_argument( - '--registry_id', - required=True, - help='Cloud IoT Core registry id') + '--registry_id', required=True, help='Cloud IoT Core registry id') parser.add_argument( - '--device_id', - required=True, - help='Cloud IoT Core device id') + '--device_id', required=True, help='Cloud IoT Core device id') parser.add_argument( '--private_key_file', - required=True, - help='Path to private key file.') + required=True, help='Path to private key file.') parser.add_argument( '--algorithm', choices=('RS256', 'ES256'), @@ -136,12 +109,19 @@ def parse_command_line_args(): parser.add_argument( '--ca_certs', default='roots.pem', - help=('CA root certificate from https://pki.google.com/roots.pem')) + help=('CA root from https://pki.google.com/roots.pem')) parser.add_argument( '--num_messages', type=int, default=100, help='Number of messages to publish.') + parser.add_argument( + '--message_type', + choices=('event', 'state'), + default='event', + required=True, + help=('Indicates whether the message to be published is a ' + 'telemetry event or a device state message.')) parser.add_argument( '--mqtt_bridge_hostname', default='mqtt.googleapis.com', @@ -158,11 +138,12 @@ def main(): # Create our MQTT client. The client_id is a unique string that identifies # this device. For Google Cloud IoT Core, it must be in the format below. client = mqtt.Client( - client_id=( - 'projects/{}/locations/{}/registries/{}/devices/{}' - .format( - args.project_id, args.cloud_region, - args.registry_id, args.device_id))) + client_id=('projects/{}/locations/{}/registries/{}/devices/{}' + .format( + args.project_id, + args.cloud_region, + args.registry_id, + args.device_id))) # With Google Cloud IoT Core, the username field is ignored, and the # password field is used to transmit a JWT to authorize the device. @@ -187,20 +168,24 @@ def main(): # Start the network loop. client.loop_start() - mqtt_topic = '/devices/{}/events'.format(args.device_id) + # Publish to the events or state topic based on the flag. + sub_topic = 'events' if args.message_type == 'event' else 'state' + + mqtt_topic = '/devices/{}/{}'.format(args.device_id, sub_topic) # Publish num_messages mesages to the MQTT bridge once per second. for i in range(1, args.num_messages + 1): payload = '{}/{}-payload-{}'.format( args.registry_id, args.device_id, i) - print( - 'Publishing message {}/{}: \'{}\''.format( - i, args.num_messages, payload)) + print('Publishing message {}/{}: \'{}\''.format( + i, args.num_messages, payload)) # Publish "payload" to the MQTT topic. qos=1 means at least once # delivery. Cloud IoT Core also supports qos=0 for at most once # delivery. client.publish(mqtt_topic, payload, qos=1) - time.sleep(1) + + # Send events every second. State should not be updated as often + time.sleep(1 if args.message_type == 'event' else 5) # End the network loop and finish. client.loop_stop()