Merge branch 'devel'

james133 · Apr 26, 2020 · 4be239e · 4be239e
2 parents f46e685 + 7c23d0f
commit 4be239e
Show file tree

Hide file tree

Showing 72 changed files with 2,668 additions and 725 deletions.
diff --git a/.github/issue_template.md b/.github/issue_template.md
@@ -1,3 +1,5 @@
+# If you are not reporting a bug or requesting a feature, please post to https://groups.google.com/d/forum/tinode instead.
+
 ### Subject of the issue
 Describe your issue here.
 
@@ -12,7 +14,7 @@ Describe your issue here.
   * platform (Windows, Mac, Linux etc)
   * version of tinode server, e.g. `0.15.2-rc3`
   * database backend
-  
+
 #### Client-side
 - [ ] TinodeWeb/tinodejs: javascript client
   * Browser make and version.

diff --git a/README.md b/README.md
@@ -75,9 +75,9 @@ When you register a new account you are asked for an email address to send valid
 * Persistent message store, paginated message history.
 * Javascript bindings with no external dependencies.
 * Java bindings (dependencies: [Jackson](https://github.com/FasterXML/jackson), [Java-Websocket](https://github.com/TooTallNate/Java-WebSocket)). Suitable for Android but with no Android SDK dependencies.
-* Websocket, long polling, and [gRPC](https://grpc.io/) over TCP transports.
+* Websocket, long polling, and [gRPC](https://grpc.io/) over TCP or Unix sockets.
 * JSON or [protobuf version 3](https://developers.google.com/protocol-buffers/) wire protocols.
-* [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security) with [Letsencrypt](https://letsencrypt.org/) or conventional certificates.
+* Optional built-in [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security) with [Letsencrypt](https://letsencrypt.org/) or conventional certificates.
 * User search/discovery.
 * Rich formatting of messages, markdown-style: \*style\* &rarr; **style**.
 * Inline images and file attachments.
@@ -86,9 +86,9 @@ When you register a new account you are asked for an email address to send valid
 * Support for client-side data caching.
 * Ability to block unwanted communication server-side.
 * Anonymous users (important for use cases related to tech support over chat).
-* Push notifications using [FCM](https://firebase.google.com/docs/cloud-messaging/).
+* Push notifications using [FCM](https://firebase.google.com/docs/cloud-messaging/) or [TNPG](server/push/tnpg/).
 * Storage and out of band transfer of large objects like video files using local file system or Amazon S3.
-* Plugins to extend functionality, for example to enable chatbots.
+* Plugins to extend functionality, for example, to enable chatbots.
 
 ### Planned
 

diff --git a/chatbot/python/chatbot.py b/chatbot/python/chatbot.py
@@ -6,6 +6,7 @@
 import argparse
 import base64
 from concurrent import futures
+from datetime import datetime
 import json
 import os
 import pkg_resources
@@ -20,15 +21,19 @@
 import time
 
 import grpc
+from google.protobuf.json_format import MessageToDict
 
 # Import generated grpc modules
 from tinode_grpc import pb
 from tinode_grpc import pbx
 
 APP_NAME = "Tino-chatbot"
-APP_VERSION = "1.1.3"
+APP_VERSION = "1.2.0"
 LIB_VERSION = pkg_resources.get_distribution("tinode_grpc").version
 
+# Maximum length of string to log. Shorten longer strings.
+MAX_LOG_LEN = 64
+
 # User ID of the current user
 botUID = None
 
@@ -38,20 +43,36 @@
 # This is needed for gRPC ssl to work correctly.
 os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
 
+def log(*args):
+    print(datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3], *args)
+
 # Add bundle for future execution
 def add_future(tid, bundle):
     onCompletion[tid] = bundle
 
+# Shorten long strings for logging.
+class JsonHelper(json.JSONEncoder):
+    def default(self, obj):
+        if type(obj) == str and len(obj) > MAX_LOG_LEN:
+            return '<' + len(obj) + ', bytes: ' + obj[:12] + '...' + obj[-12:] + '>'
+        return super(JsonHelper, self).default(obj)
+
 # Resolve or reject the future
 def exec_future(tid, code, text, params):
     bundle = onCompletion.get(tid)
     if bundle != None:
         del onCompletion[tid]
-        if code >= 200 and code < 400:
-            arg = bundle.get('arg')
-            bundle.get('action')(arg, params)
-        else:
-            print("Error:", code, text)
+        try:
+            if code >= 200 and code < 400:
+                arg = bundle.get('arg')
+                bundle.get('onsuccess')(arg, params)
+            else:
+                log("Error: {} {} ({})".format(code, text, tid))
+                onerror = bundle.get('onerror')
+                if onerror:
+                    onerror(bundle.get('arg'), {'code': code, 'text': text})
+        except Exception as err:
+            log("Error handling server response", err)
 
 # List of active subscriptions
 subscriptions = {}
@@ -61,10 +82,24 @@ def add_subscription(topic):
 def del_subscription(topic):
     subscriptions.pop(topic, None)
 
+def subscription_failed(topic, errcode):
+    if topic == 'me':
+        # Failed 'me' subscription means the bot is disfunctional.
+        if errcode.get('code') == 502:
+            # Cluster unreachable. Break the loop and retry in a few seconds.
+            client_post(None)
+        else:
+            exit(1)
+
+def login_error(unused, errcode):
+    # Check for 409 "already authenticated".
+    if errcode.get('code') != 409:
+        exit(1)
+
 def server_version(params):
     if params == None:
         return
-    print("Server:", params['build'].decode('ascii'), params['ver'].decode('ascii'))
+    log("Server:", params['build'].decode('ascii'), params['ver'].decode('ascii'))
 
 def next_id():
     next_id.tid += 1
@@ -98,7 +133,7 @@ def Account(self, acc_event, context):
         else:
             action = "unknown"
 
-        print("Account", action, ":", acc_event.user_id, acc_event.public)
+        log("Account", action, ":", acc_event.user_id, acc_event.public)
 
         return pb.Unused()
 
@@ -109,7 +144,7 @@ def client_generate():
         msg = queue_out.get()
         if msg == None:
             return
-        # print("out:", msg)
+        log("out:", json.dumps(MessageToDict(msg), cls=JsonHelper))
         yield msg
 
 def client_post(msg):
@@ -126,7 +161,7 @@ def client_reset():
 def hello():
     tid = next_id()
     add_future(tid, {
-        'action': lambda unused, params: server_version(params),
+        'onsuccess': lambda unused, params: server_version(params),
     })
     return pb.ClientMsg(hi=pb.ClientHi(id=tid, user_agent=APP_NAME + "/" + APP_VERSION + " (" +
         platform.system() + "/" + platform.release() + "); gRPC-python/" + LIB_VERSION,
@@ -136,23 +171,25 @@ def login(cookie_file_name, scheme, secret):
     tid = next_id()
     add_future(tid, {
         'arg': cookie_file_name,
-        'action': lambda fname, params: on_login(fname, params),
+        'onsuccess': lambda fname, params: on_login(fname, params),
+        'onerror': lambda unused, errcode: login_error(unused, errcode),
     })
     return pb.ClientMsg(login=pb.ClientLogin(id=tid, scheme=scheme, secret=secret))
 
 def subscribe(topic):
     tid = next_id()
     add_future(tid, {
         'arg': topic,
-        'action': lambda topicName, unused: add_subscription(topicName),
+        'onsuccess': lambda topicName, unused: add_subscription(topicName),
+        'onerror': lambda topicName, errcode: subscription_failed(topicName, errcode),
     })
     return pb.ClientMsg(sub=pb.ClientSub(id=tid, topic=topic))
 
 def leave(topic):
     tid = next_id()
     add_future(tid, {
         'arg': topic,
-        'action': lambda topicName, unused: del_subscription(topicName)
+        'onsuccess': lambda topicName, unused: del_subscription(topicName)
     })
     return pb.ClientMsg(leave=pb.ClientLeave(id=tid, topic=topic))
 
@@ -171,12 +208,12 @@ def init_server(listen):
     server.add_insecure_port(listen)
     server.start()
 
-    print("Plugin server running at '"+listen+"'")
+    log("Plugin server running at '"+listen+"'")
 
     return server
 
 def init_client(addr, schema, secret, cookie_file_name, secure, ssl_host):
-    print("Connecting to", "secure" if secure else "", "server at", addr,
+    log("Connecting to", "secure" if secure else "", "server at", addr,
         "SNI="+ssl_host if ssl_host else "")
 
     channel = None
@@ -200,13 +237,15 @@ def client_message_loop(stream):
     try:
         # Read server responses
         for msg in stream:
-            # print("in:", msg)
+            log(datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3],
+                "in:", json.dumps(MessageToDict(msg), cls=JsonHelper))
+
             if msg.HasField("ctrl"):
                 # Run code on command completion
                 exec_future(msg.ctrl.id, msg.ctrl.code, msg.ctrl.text, msg.ctrl.params)
 
             elif msg.HasField("data"):
-                # print("message from:", msg.data.from_user_id)
+                # log("message from:", msg.data.from_user_id)
 
                 # Protection against the bot talking to self from another session.
                 if msg.data.from_user_id != botUID:
@@ -219,7 +258,7 @@ def client_message_loop(stream):
                     client_post(publish(msg.data.topic, next_quote()))
 
             elif msg.HasField("pres"):
-                # print("presence:", msg.pres.topic, msg.pres.what)
+                # log("presence:", msg.pres.topic, msg.pres.what)
                 # Wait for peers to appear online and subscribe to their topics
                 if msg.pres.topic == 'me':
                     if (msg.pres.what == pb.ServerPres.ON or msg.pres.what == pb.ServerPres.MSG) \
@@ -233,7 +272,7 @@ def client_message_loop(stream):
                 pass
 
     except grpc._channel._Rendezvous as err:
-        print("Disconnected:", err)
+        log("Disconnected:", err)
 
 def read_auth_cookie(cookie_file_name):
     """Read authentication token from a file"""
@@ -272,7 +311,7 @@ def on_login(cookie_file_name, params):
         json.dump(nice, cookie)
         cookie.close()
     except Exception as err:
-        print("Failed to save authentication cookie", err)
+        log("Failed to save authentication cookie", err)
 
 def load_quotes(file_name):
     with open(file_name) as f:
@@ -289,25 +328,25 @@ def run(args):
         """Use token to login"""
         schema = 'token'
         secret = args.login_token.encode('acsii')
-        print("Logging in with token", args.login_token)
+        log("Logging in with token", args.login_token)
 
     elif args.login_basic:
         """Use username:password"""
         schema = 'basic'
         secret = args.login_basic.encode('utf-8')
-        print("Logging in with login:password", args.login_basic)
+        log("Logging in with login:password", args.login_basic)
 
     else:
         """Try reading the cookie file"""
         try:
             schema, secret = read_auth_cookie(args.login_cookie)
-            print("Logging in with cookie file", args.login_cookie)
+            log("Logging in with cookie file", args.login_cookie)
         except Exception as err:
-            print("Failed to read authentication cookie", err)
+            log("Failed to read authentication cookie", err)
 
     if schema:
         # Load random quotes from file
-        print("Loaded {} quotes".format(load_quotes(args.quotes)))
+        log("Loaded {} quotes".format(load_quotes(args.quotes)))
 
         # Start Plugin server
         server = init_server(args.listen)
@@ -317,7 +356,7 @@ def run(args):
 
         # Setup closure for graceful termination
         def exit_gracefully(signo, stack_frame):
-            print("Terminated with signal", signo)
+            log("Terminated with signal", signo)
             server.stop(0)
             client.cancel()
             sys.exit(0)
@@ -339,17 +378,17 @@ def exit_gracefully(signo, stack_frame):
         client.cancel()
 
     else:
-        print("Error: authentication scheme not defined")
+        log("Error: authentication scheme not defined")
 
 
 if __name__ == '__main__':
     """Parse command-line arguments. Extract server host name, listen address, authentication scheme"""
     random.seed()
 
     purpose = "Tino, Tinode's chatbot."
-    print(purpose)
+    log(purpose)
     parser = argparse.ArgumentParser(description=purpose)
-    parser.add_argument('--host', default='localhost:6061', help='address of Tinode server gRPC endpoint')
+    parser.add_argument('--host', default='localhost:16060', help='address of Tinode server gRPC endpoint')
     parser.add_argument('--ssl', action='store_true', help='use SSL to connect to the server')
     parser.add_argument('--ssl-host', help='SSL host name to use instead of default (useful for connecting to localhost)')
     parser.add_argument('--listen', default='0.0.0.0:40051', help='address to listen on for incoming Plugin API calls')

diff --git a/chatbot/python/requirements.txt b/chatbot/python/requirements.txt
@@ -1,4 +1,3 @@
 futures>=3.2.0; python_version<'3'
 grpcio>=1.18.0
-requests>=2.21.0
-tinode-grpc>=0.15.11
+tinode-grpc>=0.16
diff --git a/docker-build.sh b/docker-build.sh
@@ -28,7 +28,7 @@ for dbtag in "${dbtags[@]}"
 do
   if [ "$dbtag" == "alldbs" ]; then
     # For alldbs, container name is tinode/tinode.
-    name="tiniode/tinode"
+    name="tinode/tinode"
   else
     # Otherwise, tinode/tinode-$dbtag.
     name="tinode/tinode-${dbtag}"
@@ -53,3 +53,13 @@ if [ -n "$FULLRELEASE" ]; then
 fi
 docker rmi ${rmitags}
 docker build --build-arg VERSION=$tag ${buildtags} docker/chatbot
+
+# Build exporter image
+buildtags="--tag tinode/exporter:${ver[0]}.${ver[1]}.${ver[2]}"
+rmitags="tinode/exporter:${ver[0]}.${ver[1]}.${ver[2]}"
+if [ -n "$FULLRELEASE" ]; then
+  rmitags="${rmitags} tinode/exporter:latest tinode/exporter:${ver[0]}.${ver[1]}"
+  buildtags="${buildtags}  --tag tinode/exporter:latest --tag tinode/exporter:${ver[0]}.${ver[1]}"
+fi
+docker rmi ${rmitags}
+docker build --build-arg VERSION=$tag ${buildtags} docker/exporter
diff --git a/docker-release.sh b/docker-release.sh
@@ -39,30 +39,6 @@ source .dockerhub
 # Login to docker hub
 docker login -u $user -p $pass
 
-# Remove earlier builds
-for dbtag in "${dbtags[@]}"
-do
-  name="$(containerName $dbtag)"
-  if [ -n "$FULLRELEASE" ]; then
-    curl -u $user:$pass -i -X DELETE \
-      https://cloud.docker.com/v2/repositories/tinode/${name}/tags/latest/
-
-    curl -u $user:$pass -i -X DELETE \
-      https://cloud.docker.com/v2/repositories/tinode/${name}/tags/${ver[0]}.${ver[1]}/
-  fi
-  curl -u $user:$pass -i -X DELETE \
-    https://cloud.docker.com/v2/repositories/tinode/${name}/tags/${ver[0]}.${ver[1]}.${ver[2]}/
-done
-
-if [ -n "$FULLRELEASE" ]; then
-  curl -u $user:$pass -i -X DELETE \
-    https://cloud.docker.com/v2/repositories/tinode/chatbot/tags/latest/
-  curl -u $user:$pass -i -X DELETE \
-    https://cloud.docker.com/v2/repositories/tinode/chatbot/tags/${ver[0]}.${ver[1]}/
-fi
-curl -u $user:$pass -i -X DELETE \
-  https://cloud.docker.com/v2/repositories/tinode/chatbot/tags/${ver[0]}.${ver[1]}.${ver[2]}/
-
 # Deploy images for various DB backends
 for dbtag in "${dbtags[@]}"
 do
@@ -82,4 +58,11 @@ if [ -n "$FULLRELEASE" ]; then
 fi
 docker push tinode/chatbot:"${ver[0]}.${ver[1]}.${ver[2]}"
 
+# Deploy exporter images
+if [ -n "$FULLRELEASE" ]; then
+  docker push tinode/exporter:latest
+  docker push tinode/exporter:"${ver[0]}.${ver[1]}"
+fi
+docker push tinode/exporter:"${ver[0]}.${ver[1]}.${ver[2]}"
+
 docker logout