|
| 1 | +# |
| 2 | +# Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | +# contributor license agreements. See the NOTICE file distributed with |
| 4 | +# this work for additional information regarding copyright ownership. |
| 5 | +# The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | +# (the "License"); you may not use this file except in compliance with |
| 7 | +# the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | +# |
| 17 | + |
| 18 | +""" |
| 19 | + A sample wordcount with MqttStream stream |
| 20 | + Usage: mqtt_wordcount.py <broker url> <topic> |
| 21 | + To work with Mqtt, Mqtt Message broker/server required. |
| 22 | + Mosquitto (http://mosquitto.org/) is an open source Mqtt Broker |
| 23 | + In ubuntu mosquitto can be installed using the command `$ sudo apt-get install mosquitto` |
| 24 | + Run Mqtt publisher as |
| 25 | + `$ bin/run-example \ |
| 26 | + org.apache.spark.examples.streaming.MQTTPublisher tcp://localhost:1883 foo` |
| 27 | + and then run the example as |
| 28 | + `$ bin/spark-submit --driver-class-path external/mqtt-assembly/target/scala-*/\ |
| 29 | + spark-streaming-mqtt-assembly-*.jar examples/src/main/python/streaming/mqtt_wordcount.py \ |
| 30 | + tcp://localhost:1883 foo` |
| 31 | +""" |
| 32 | + |
| 33 | +import sys |
| 34 | + |
| 35 | +from pyspark import SparkContext |
| 36 | +from pyspark.streaming import StreamingContext |
| 37 | +from pyspark.streaming.mqtt import MQTTUtils |
| 38 | + |
| 39 | +if __name__ == "__main__": |
| 40 | + if len(sys.argv) != 3: |
| 41 | + print >> sys.stderr, "Usage: mqtt_wordcount.py <broker url> <topic>" |
| 42 | + exit(-1) |
| 43 | + |
| 44 | + sc = SparkContext(appName="PythonStreamingMQTTWordCount") |
| 45 | + ssc = StreamingContext(sc, 1) |
| 46 | + |
| 47 | + broker_url = sys.argv[1] |
| 48 | + topic = sys.argv[2] |
| 49 | + |
| 50 | + data = MQTTUtils.createStream(ssc, topic, broker_url) |
| 51 | + lines = data.map(lambda x: x[1]) |
| 52 | + counts = lines.flatMap(lambda line: line.split(" ")) \ |
| 53 | + .map(lambda word: (word, 1)) \ |
| 54 | + .reduceByKey(lambda a, b: a+b) |
| 55 | + counts.pprint() |
| 56 | + |
| 57 | + ssc.start() |
| 58 | + ssc.awaitTermination() |
0 commit comments