Work in progress.

alex-requirements.txt

@@ -23,3 +23,7 @@ sphinxcontrib-napoleon
 
 flask
 theano
+
+twisted
+autobahn
+protobuf

alex/applications/PublicTransportInfoCS/wshub_lz.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+pushd ..
+
+./wshub.py -c ./PublicTransportInfoCS/ptics.cfg ../resources/private/ext-lz-277278190.cfg ./PublicTransportInfoCS/kaldi.cfg ./PublicTransportInfoCS/ptics_hdc_slu.cfg
+
+
+popd

alex/applications/wshub.py

@@ -12,7 +12,7 @@
 from alex.components.hub.vad import VAD
 from alex.components.hub.asr import ASR
 from alex.components.hub.slu import SLU
-from alex.components.hub.dm import DM
+from alex.components.hub.dm import DM, DMDA
 from alex.components.hub.nlg import NLG
 from alex.components.hub.tts import TTS
 from alex.components.hub.messages import Command
@@ -103,7 +103,11 @@ def run(self):
         call_back_uri = None
         call_start = time.time()
 
-        self.cfg['Logging']['system_logger'].session_start("@LOCAL_CALL")
+        self.cfg['Logging']['session_logger'].set_close_event(close_event)
+        self.cfg['Logging']['session_logger'].set_cfg(self.cfg)
+        self.cfg['Logging']['session_logger'].start()
+
+        self.cfg['Logging']['system_logger'].session_start("LOCAL_CALL")
         self.cfg['Logging']['system_logger'].session_system_log('config = ' + str(self.cfg))
 
         self.cfg['Logging']['session_logger'].session_start(
@@ -113,14 +117,20 @@ def run(self):
             self.cfg['Logging']["system_name"], self.cfg['Logging']["version"])
         self.cfg['Logging']['session_logger'].input_source("aio")
 
+
+
         while 1:
             time.sleep(self.cfg['Hub']['main_loop_sleep_time'])
 
-            if call_back_time != -1 and call_back_time < time.time():
-                aio_commands.send(Command('make_call(destination="%s")' %
-                                          call_back_uri, 'HUB', 'AIO'))
-                call_back_time = -1
-                call_back_uri = None
+            # read all messages
+            if aio_commands.poll():
+                command = aio_commands.recv()
+                self.cfg['Logging']['system_logger'].info(command)
+
+                if isinstance(command, Command):
+                    if command.parsed['__name__'] == "client_connected":
+                        dm_commands.send(Command('new_dialogue()', 'HUB', 'DM'))
+
 
             if vad_commands.poll():
                 command = vad_commands.recv()
@@ -147,11 +157,20 @@ def run(self):
                 if isinstance(command, Command):
                     if command.parsed['__name__'] == "hangup":
                         # prepare for ending the call
-                        pass
+                        hangup = True
+                        self.cfg['Analytics'].track_event('vhub', 'system_hangup')
 
-                    if command.parsed['__name__'] == "dm_da_generated":
-                        # record the time of the last system generated dialogue act
-                        pass
+                    if command.parsed['__name__'] == "flushed":
+                        # flush nlg, when flushed, tts will be flushed
+                        nlg_commands.send(Command('flush()', 'HUB', 'NLG'))
+
+                elif isinstance(command, DMDA):
+                    # record the time of the last system generated dialogue act
+                    s_last_dm_activity_time = time.time()
+
+                    if command.da != "silence()":
+                        # if the DM generated non-silence dialogue act, then continue in processing it
+                        nlg_commands.send(DMDA(command.da, "HUB", "NLG"))
 
             if nlg_commands.poll():
                 command = nlg_commands.recv()

alex/components/hub/vad.py

@@ -166,6 +166,8 @@ def read_write_audio(self):
                 decision = self.vad.decide(data_rec.payload)
                 vad, change = self.smoothe_decison(decision)
 
+                if change:
+                    print change
                 #d = (time.time() - s[0], time.clock() - s[1])
                 #if d[0] > 0.001:
                 #    print "VAD t = {t:0.4f} c = {c:0.4f}\n".format(t=d[0], c=d[1])

alex/components/hub/wsio.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import pyaudio
 from jinja2.loaders import FileSystemLoader
 from jinja2 import Environment
 
@@ -13,7 +12,7 @@
 import urlparse
 import Queue
 import BaseHTTPServer
-from threading import Thread
+import threading
 import time
 
 from alex.utils.audio import load_wav
@@ -48,7 +47,7 @@ def __init__(self, cfg, commands, audio_record, audio_play, close_event):
         self.audio_play = audio_play
         self.close_event = close_event
 
-    def process_pending_commands(self, p, stream, wf):
+    def process_pending_commands(self):
         """Process all pending commands.
 
         Available commands:
@@ -73,12 +72,6 @@ def process_pending_commands(self, p, stream, wf):
                     while self.audio_play.poll():
                         self.audio_play.recv()
 
-                    # stop recording and playing
-                    stream.stop_stream()
-                    stream.close()
-                    p.terminate()
-                    wf.close()
-
                     return True
 
                 if command.parsed['__name__'] == 'flush':
@@ -90,7 +83,7 @@ def process_pending_commands(self, p, stream, wf):
 
         return False
 
-    def read_write_audio(self, p, stream, wf, play_buffer):
+    def read_write_audio(self): #, p, stream, wf, play_buffer):
         """Send some of the available data to the output.
         It should be a non-blocking operation.
 
@@ -99,26 +92,30 @@ def read_write_audio(self, p, stream, wf, play_buffer):
           2) send only if stream.get_write_available() is more then the frame size
         """
         if self.audio_play.poll():
-            while self.audio_play.poll() \
-                and len(play_buffer) < self.cfg['AudioIO']['play_buffer_size'] \
-                    and stream.get_write_available() > self.cfg['Audio']['samples_per_frame']:
+            while self.audio_play.poll(): # \
+                #and len(play_buffer) < self.cfg['AudioIO']['play_buffer_size']:
 
                 # send to play frames from input
                 data_play = self.audio_play.recv()
                 if isinstance(data_play, Frame):
-                    stream.write(data_play.payload)
-
-                    play_buffer.append(data_play)
-
-                    if self.cfg['AudioIO']['debug']:
-                        print '.',
-                        sys.stdout.flush()
-
-                elif isinstance(data_play, Command):
-                    if data_play.parsed['__name__'] == 'utterance_start':
-                        self.commands.send(Command('play_utterance_start()', 'AudioIO', 'HUB'))
-                    if data_play.parsed['__name__'] == 'utterance_end':
-                        self.commands.send(Command('play_utterance_end()', 'AudioIO', 'HUB'))
+                    msg = AlexToClient()
+                    msg.speech.body = data_play.payload
+                    self.ws_conn.send(self.ws_protocol, msg.SerializeToString())
+
+                #if isinstance(data_play, Frame):
+                #    stream.write(data_play.payload)
+                #
+                #    play_buffer.append(data_play)
+                #
+                #    if self.cfg['AudioIO']['debug']:
+                #        print '.',
+                #        sys.stdout.flush()
+
+                #elif isinstance(data_play, Command):
+                #    if data_play.parsed['__name__'] == 'utterance_start':
+                #        self.commands.send(Command('play_utterance_start()', 'AudioIO', 'HUB'))
+                #    if data_play.parsed['__name__'] == 'utterance_end':
+                #        self.commands.send(Command('play_utterance_end()', 'AudioIO', 'HUB'))
 
     def run(self):
         try:
@@ -127,15 +124,21 @@ def run(self):
             global logger
             logger = self.cfg['Logging']['system_logger']
 
-            factory = WebSocketServerFactory("ws://0.0.0.0:9000", debug=False)
-            factory.protocol = AlexWebsocketProtocol
+            #factory = WebSocketServerFactory("ws://0.0.0.0:9000", debug=False)
+            #factory.protocol = create_alex_websocket_protocol(self)
 
-            def run_ws():
-                reactor.listenTCP(9000, factory)
-                reactor.run()
+            #def run_ws():
+            #    print 'running ws'
+            #    reactor.listenTCP(9000, factory)
+            #    reactor.run(installSignalHandlers=0)
 
-            t = Thread(target=lambda *args: run_ws)
-            t.start()
+            #t = Thread(target=run_ws) #lambda *args: run_ws())
+            #t.setDaemon(True)
+            #print 'starting thread'
+            #t.start()
+            self.ws_conn = Connection(self)
+            self.ws_conn.daemon = True
+            self.ws_conn.start()
 
             # process incoming audio play and send requests
             while 1:
@@ -144,47 +147,81 @@ def run_ws():
                 if self.close_event.is_set():
                     return
 
+                #import ipdb; ipdb.set_trace()
+
                 # process all pending commands
-                if self.process_pending_commands(p, stream, wf):
+                if self.process_pending_commands():
                     return
 
+                print '.'
+
                 # process each web request
-                while not self.web_queue.empty():
-                    for filename in self.web_queue.get():
-                        try:
-                            self.send_wav(filename, stream)
-                        except:
-                            self.cfg['Logging']['system_logger'].exception(
-                                'Error processing file: ' + filename)
-
-                # process audio data
-                self.read_write_audio(p, stream, wf, play_buffer)
+                #while not self.web_queue.empty():
+                #    for filename in self.web_queue.get():
+                #        try:
+                #            self.send_wav(filename, stream)
+                #        except:
+                #            self.cfg['Logging']['system_logger'].exception(
+                #                'Error processing file: ' + filename)
+
+                ## process audio data
+                self.read_write_audio() #p, stream, wf, play_buffer)
         except:
             self.cfg['Logging']['system_logger'].exception('Uncaught exception in VAD process.')
             self.close_event.set()
             raise
 
+    def on_client_connected(self, protocol, request):
+        self.commands.send(Command('client_connected()', 'WSIO', 'HUB'))
+        self.ws_protocol = protocol
+
 from twisted.internet import reactor
 from autobahn.twisted.websocket import WebSocketServerProtocol, \
     WebSocketServerFactory
 
+#from wshub_messages_pb2
+from wsio_messages_pb2 import ClientToAlex, AlexToClient
+
+
+class Connection(threading.Thread):
+    def __init__(self, hub_instance):
+        super(Connection, self).__init__()
+        self.factory=WebSocketServerFactory("ws://localhost:9000", debug=True)
+        self.hub_instance = hub_instance
+
+    def run(self):
+        self.factory.protocol = create_ws_protocol(self.hub_instance)
+        reactor.listenTCP(9000, self.factory)
+        reactor.run(installSignalHandlers=0)
+
+    def send(self, proto, data):
+        reactor.callFromThread(proto.sendMessage, data, True)
+
+
+
+
+def create_ws_protocol(hub):
+    class AlexWebsocketProtocol(WebSocketServerProtocol):
+        hub_instance = hub
+
+        def onConnect(self, request):
+            print self.factory, id(self)
+            print("Client connecting: {0}".format(request.peer))
+            self.hub_instance.on_client_connected(self, request)
+
+        def onOpen(self):
+            print("WebSocket connection open.")
 
-class AlexWebsocketProtocol(WebSocketServerProtocol):
+        def onMessage(self, payload, isBinary):
+            if isBinary:
+                msg = ClientToAlex()
+                msg.ParseFromString(payload)
+                self.hub_instance.audio_record.send(Frame(msg.speech.body))
 
-    def onConnect(self, request):
-        print("Client connecting: {0}".format(request.peer))
 
-    def onOpen(self):
-        print("WebSocket connection open.")
+        def onClose(self, wasClean, code, reason):
+            print("WebSocket connection closed: {0}".format(reason))
 
-    def onMessage(self, payload, isBinary):
-        if isBinary:
-            print("Binary message received: {0} bytes".format(len(payload)))
-        else:
-            print("Text message received: {0}".format(payload.decode('utf8')))
+    return AlexWebsocketProtocol
 
-        # echo back message verbatim
-        self.sendMessage(payload, isBinary)
 
-    def onClose(self, wasClean, code, reason):
-        print("WebSocket connection closed: {0}".format(reason))