From f7816b8d69a823ecb556f74040f95b52339b3d95 Mon Sep 17 00:00:00 2001 From: Gene Date: Thu, 7 Nov 2019 16:15:26 +0300 Subject: [PATCH 01/13] Adding mongodb build tag --- server/.golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/server/.golangci.yml b/server/.golangci.yml index 68ebcb2e8..f5f72c38b 100644 --- a/server/.golangci.yml +++ b/server/.golangci.yml @@ -40,3 +40,4 @@ run: build-tags: - mysql - rethinkdb + - mongodb From 6fd59c267b11f8faaa997b6296f403b16f98064c Mon Sep 17 00:00:00 2001 From: or-else Date: Thu, 14 Nov 2019 13:05:17 +0300 Subject: [PATCH 02/13] try to catch cases when repote subscription is missing --- server/cluster.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/cluster.go b/server/cluster.go index ab93d853c..99b5d5f24 100644 --- a/server/cluster.go +++ b/server/cluster.go @@ -536,9 +536,8 @@ func (c *Cluster) routeToTopic(msg *ClientComMessage, topic string, sess *Sessio return errors.New("attempt to route to non-existent node") } - if sess.getRemoteSub(topic) == nil { - log.Printf("Remote subscription missing for topic '%s', sid '%s'", topic, sess.sid) - sess.addRemoteSub(topic, &RemoteSubscription{node: n.name}) + if sess.getRemoteSub(topic) == nil && msg.sub == nil { + panic("ERROR: Remote subscription missing for topic '%s', sid '%s'", topic, sess.sid) } req := &ClusterReq{ From cda3acb98f4bce30f616675ff253456b3a8635f6 Mon Sep 17 00:00:00 2001 From: or-else Date: Thu, 14 Nov 2019 13:11:44 +0300 Subject: [PATCH 03/13] remove panic --- server/cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/cluster.go b/server/cluster.go index 99b5d5f24..2a687a3ff 100644 --- a/server/cluster.go +++ b/server/cluster.go @@ -536,8 +536,8 @@ func (c *Cluster) routeToTopic(msg *ClientComMessage, topic string, sess *Sessio return errors.New("attempt to route to non-existent node") } - if sess.getRemoteSub(topic) == nil && msg.sub == nil { - panic("ERROR: Remote subscription missing for topic '%s', sid '%s'", topic, sess.sid) + if sess.getRemoteSub(topic) == nil { + log.Println("No remote subscription (yet) for topic '%s', sid '%s'", topic, sess.sid) } req := &ClusterReq{ From b2370c4f410f2eb994afadca4491339d4695db84 Mon Sep 17 00:00:00 2001 From: or-else Date: Thu, 14 Nov 2019 15:37:57 +0300 Subject: [PATCH 04/13] fix logging --- server/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cluster.go b/server/cluster.go index 2a687a3ff..e0e56cffa 100644 --- a/server/cluster.go +++ b/server/cluster.go @@ -537,7 +537,7 @@ func (c *Cluster) routeToTopic(msg *ClientComMessage, topic string, sess *Sessio } if sess.getRemoteSub(topic) == nil { - log.Println("No remote subscription (yet) for topic '%s', sid '%s'", topic, sess.sid) + log.Printf("No remote subscription (yet) for topic '%s', sid '%s'", topic, sess.sid) } req := &ClusterReq{ From a116bc24a26a5c4abfbe9300bded7a541a4e734a Mon Sep 17 00:00:00 2001 From: or-else Date: Fri, 15 Nov 2019 11:10:43 +0300 Subject: [PATCH 05/13] simple tooling for debugging threading issues --- server/main.go | 9 +++++++-- server/threadz.go | 33 +++++++++++++++++++++++++++++++++ server/tinode.conf | 2 +- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 server/threadz.go diff --git a/server/main.go b/server/main.go index 74306f901..52c03acfb 100644 --- a/server/main.go +++ b/server/main.go @@ -238,13 +238,14 @@ func main() { var configfile = flag.String("config", "tinode.conf", "Path to config file.") // Path to static content. - var staticPath = flag.String("static_data", defaultStaticPath, "Path to directory with static files to be served.") + var staticPath = flag.String("static_data", defaultStaticPath, "File path to directory with static files to be served.") var listenOn = flag.String("listen", "", "Override address and port to listen on for HTTP(S) clients.") var listenGrpc = flag.String("grpc_listen", "", "Override address and port to listen on for gRPC clients.") var tlsEnabled = flag.Bool("tls_enabled", false, "Override config value for enabling TLS.") var clusterSelf = flag.String("cluster_self", "", "Override the name of the current cluster node.") - var expvarPath = flag.String("expvar", "", "Override the path where runtime stats are exposed.") + var expvarPath = flag.String("expvar", "", "Override the URL path where runtime stats are exposed. Use '-' to disable.") var pprofFile = flag.String("pprof", "", "File name to save profiling info to. Disabled if not set.") + var threadzPath = flag.String("threadz", "", "Debugging only! URL path for exposing call stacks of all goroutines.") flag.Parse() *configfile = toAbsolutePath(rootpath, *configfile) @@ -264,6 +265,7 @@ func main() { // Set up HTTP server. Must use non-default mux because of expvar. mux := http.NewServeMux() + // Exposing values for statistics and monitoring. evpath := *expvarPath if evpath == "" { evpath = config.ExpvarPath @@ -272,6 +274,9 @@ func main() { statsRegisterInt("Version") statsSet("Version", int64(parseVersion(currentVersion))) + // Initialize optional debug stack tracing. + threadzInit(mux, *threadzPath) + // Initialize cluster and receive calculated workerId. // Cluster won't be started here yet. workerId := clusterInit(config.Cluster, clusterSelf) diff --git a/server/threadz.go b/server/threadz.go new file mode 100644 index 000000000..2037ca655 --- /dev/null +++ b/server/threadz.go @@ -0,0 +1,33 @@ +// Debugging threading issues: dumps stacktraces of all goroutines in response to +// HTTP request to the given path. + +package main + +import ( + "log" + "net/http" + "runtime/pprof" + "strings" +) + +// Initialize stack tracing at the given URL path. +func threadzInit(mux *http.ServeMux, path string) { + if path == "" || path == "-" { + return + } + + // Make sure the path is absolute. + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + + mux.HandleFunc(path, threadzHandler) + + log.Printf("threadz: stack traces exposed at '%s'", path) +} + +// Handler which dumps stacktraces of all goroutines to response writer. +func threadzHandler(wrt http.ResponseWriter, req *http.Request) { + wrt.Header().Set("Content-Type", "text/plain; charset=utf-8") + pprof.Lookup("goroutine").WriteTo(wrt, 2) +} diff --git a/server/tinode.conf b/server/tinode.conf index 2a3ca9df7..605c0fe5b 100644 --- a/server/tinode.conf +++ b/server/tinode.conf @@ -36,7 +36,7 @@ // Maximum number of indexable tags per topic or user. "max_tag_count": 16, - // URL path for exposing runtime stats. Disabled if the path is blank. + // URL path for exposing runtime stats. Disabled if the path is blank or "-". // Could be overriden from the command line with --expvar. "expvar": "/debug/vars", From d12d809e1ede04fc4c949149e71912a95513430f Mon Sep 17 00:00:00 2001 From: or-else Date: Fri, 15 Nov 2019 11:11:33 +0300 Subject: [PATCH 06/13] mention second webserver as it seems to be a common problem --- INSTALL.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 1e82688c0..97b440d07 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -97,6 +97,9 @@ See [instructions](./docker/README.md) 5. Test your installation by pointing your browser to [http://localhost:6060/](http://localhost:6060/). The static files from the `-static_data` path are served at web root `/`. You can change this by editing the line `static_mount` in the config file. +If you are running Tinode alongside another webserver, such as Apache or nginx, keep in mind that you need to launch the webapp from the URL served by Tinode. Otherwise it won't work. + + ## Running a Cluster - Install and run the database, run DB initializer, unpack JS files as described in the previous section. Both MySQL and RethinkDB supports [cluster](https://www.mysql.com/products/cluster/) [mode](https://www.rethinkdb.com/docs/start-a-server/#a-rethinkdb-cluster-using-multiple-machines). You may consider it for added resiliency. From 801abb6909ba87ef3a6c9781597c85c48fa80239 Mon Sep 17 00:00:00 2001 From: or-else Date: Fri, 15 Nov 2019 15:08:52 +0300 Subject: [PATCH 07/13] allow dumping all possible profiles, not just goroutines --- server/main.go | 6 +++--- server/threadz.go | 47 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/server/main.go b/server/main.go index 52c03acfb..4b9c57307 100644 --- a/server/main.go +++ b/server/main.go @@ -245,7 +245,7 @@ func main() { var clusterSelf = flag.String("cluster_self", "", "Override the name of the current cluster node.") var expvarPath = flag.String("expvar", "", "Override the URL path where runtime stats are exposed. Use '-' to disable.") var pprofFile = flag.String("pprof", "", "File name to save profiling info to. Disabled if not set.") - var threadzPath = flag.String("threadz", "", "Debugging only! URL path for exposing call stacks of all goroutines.") + var pprofUrl = flag.String("pprof_url", "", "Debugging only! URL path for exposing profiling info. Disabled if not set.") flag.Parse() *configfile = toAbsolutePath(rootpath, *configfile) @@ -274,8 +274,8 @@ func main() { statsRegisterInt("Version") statsSet("Version", int64(parseVersion(currentVersion))) - // Initialize optional debug stack tracing. - threadzInit(mux, *threadzPath) + // Initialize serving debug profiles (optional). + servePprof(mux, *pprofUrl) // Initialize cluster and receive calculated workerId. // Cluster won't be started here yet. diff --git a/server/threadz.go b/server/threadz.go index 2037ca655..8cdcf2177 100644 --- a/server/threadz.go +++ b/server/threadz.go @@ -1,33 +1,52 @@ -// Debugging threading issues: dumps stacktraces of all goroutines in response to -// HTTP request to the given path. +// Debug tooling. Dumps named profile in response to HTTP request at +// http(s)://// +// See godoc for the list of possible profile names: https://golang.org/pkg/runtime/pprof/#Profile package main import ( + "fmt" "log" "net/http" + "path" "runtime/pprof" "strings" ) -// Initialize stack tracing at the given URL path. -func threadzInit(mux *http.ServeMux, path string) { - if path == "" || path == "-" { +var pprofHttpRoot string + +// Expose debug profiling at the given URL path. +func servePprof(mux *http.ServeMux, serveAt string) { + if serveAt == "" || serveAt == "-" { return } - // Make sure the path is absolute. - if !strings.HasPrefix(path, "/") { - path = "/" + path - } + pprofHttpRoot = path.Clean("/"+serveAt) + "/" + mux.HandleFunc(pprofHttpRoot, profileHandler) - mux.HandleFunc(path, threadzHandler) + log.Printf("pprof: profiling info exposed at '%s'", pprofHttpRoot) +} + +func profileHandler(wrt http.ResponseWriter, req *http.Request) { + wrt.Header().Set("X-Content-Type-Options", "nosniff") + wrt.Header().Set("Content-Type", "text/plain; charset=utf-8") + + profileName := strings.TrimPrefix(req.URL.Path, pprofHttpRoot) + + profile := pprof.Lookup(profileName) + if profile == nil { + servePprofError(wrt, http.StatusNotFound, "Unknown profile '"+profileName+"'") + return + } - log.Printf("threadz: stack traces exposed at '%s'", path) + // Respond with the requested profile. + profile.WriteTo(wrt, 2) } -// Handler which dumps stacktraces of all goroutines to response writer. -func threadzHandler(wrt http.ResponseWriter, req *http.Request) { +func servePprofError(wrt http.ResponseWriter, status int, txt string) { wrt.Header().Set("Content-Type", "text/plain; charset=utf-8") - pprof.Lookup("goroutine").WriteTo(wrt, 2) + wrt.Header().Set("X-Go-Pprof", "1") + wrt.Header().Del("Content-Disposition") + wrt.WriteHeader(status) + fmt.Fprintln(wrt, txt) } From 7c331cc60796c01c1abe86cdf7d8fcd597d9cb7e Mon Sep 17 00:00:00 2001 From: googlom Date: Fri, 15 Nov 2019 17:36:04 +0500 Subject: [PATCH 08/13] move store.GetUid() from adapter to store --- server/db/rethinkdb/adapter.go | 2 -- server/store/store.go | 13 +++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/server/db/rethinkdb/adapter.go b/server/db/rethinkdb/adapter.go index 9c0beb5da..4913066b0 100644 --- a/server/db/rethinkdb/adapter.go +++ b/server/db/rethinkdb/adapter.go @@ -1514,7 +1514,6 @@ func (a *adapter) FindTopics(req, opt []string) ([]t.Subscription, error) { // Messages func (a *adapter) MessageSave(msg *t.Message) error { - msg.SetUid(store.GetUid()) _, err := rdb.DB(a.dbName).Table("messages").Insert(msg).RunWrite(a.conn) return err } @@ -1653,7 +1652,6 @@ func (a *adapter) MessageDeleteList(topic string, toDel *t.DelMessage) error { err = a.messagesHardDelete(topic) } else { // Only some messages are being deleted - toDel.SetUid(store.GetUid()) // Start with making a log entry _, err = rdb.DB(a.dbName).Table("dellog").Insert(toDel).RunWrite(a.conn) diff --git a/server/store/store.go b/server/store/store.go index fdbf140ac..1ae100de9 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -160,10 +160,10 @@ func RegisterAdapter(a adapter.Adapter) { adp = a } -// GetUid generates a unique ID suitable for use as a primary key. -func GetUid() types.Uid { - return uGen.Get() -} +// GetUid generates a unique ID suitable for use as a primary key. (Not used anymore) +//func GetUid() types.Uid { +// return uGen.Get() +//} // GetUidString generate unique ID as string func GetUidString() string { @@ -197,7 +197,7 @@ var Users UsersObjMapper // Create inserts User object into a database, updates creation time and assigns UID func (UsersObjMapper) Create(user *types.User, private interface{}) (*types.User, error) { - user.SetUid(GetUid()) + user.SetUid(uGen.Get()) user.InitTimes() err := adp.UserCreate(user) @@ -510,7 +510,7 @@ var Messages MessagesObjMapper // Save message func (MessagesObjMapper) Save(msg *types.Message, readBySender bool) error { msg.InitTimes() - + msg.SetUid(uGen.Get()) // Increment topic's or user's SeqId err := adp.TopicUpdateOnMessage(msg.Topic, msg) if err != nil { @@ -571,6 +571,7 @@ func (MessagesObjMapper) DeleteList(topic string, delID int, forUser types.Uid, DelId: delID, DeletedFor: forUser.String(), SeqIdRanges: ranges} + toDel.SetUid(uGen.Get()) toDel.InitTimes() } From 0ad81118109513f449d8264532bf26a8f41bf6ac Mon Sep 17 00:00:00 2001 From: or-else Date: Fri, 15 Nov 2019 22:10:24 +0300 Subject: [PATCH 09/13] renamed threadz.go to http_pprof.go --- server/{threadz.go => http_pprof.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/{threadz.go => http_pprof.go} (100%) diff --git a/server/threadz.go b/server/http_pprof.go similarity index 100% rename from server/threadz.go rename to server/http_pprof.go From dcff3c929091f5ddd8ed6e834dae013c11b1091a Mon Sep 17 00:00:00 2001 From: googlom Date: Sat, 16 Nov 2019 11:32:19 +0500 Subject: [PATCH 10/13] GetUid fix --- server/store/store.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/store/store.go b/server/store/store.go index 1ae100de9..1c1465a09 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -160,10 +160,10 @@ func RegisterAdapter(a adapter.Adapter) { adp = a } -// GetUid generates a unique ID suitable for use as a primary key. (Not used anymore) -//func GetUid() types.Uid { -// return uGen.Get() -//} +// GetUid generates a unique ID suitable for use as a primary key. +func GetUid() types.Uid { + return uGen.Get() +} // GetUidString generate unique ID as string func GetUidString() string { @@ -197,7 +197,7 @@ var Users UsersObjMapper // Create inserts User object into a database, updates creation time and assigns UID func (UsersObjMapper) Create(user *types.User, private interface{}) (*types.User, error) { - user.SetUid(uGen.Get()) + user.SetUid(GetUid()) user.InitTimes() err := adp.UserCreate(user) @@ -510,7 +510,7 @@ var Messages MessagesObjMapper // Save message func (MessagesObjMapper) Save(msg *types.Message, readBySender bool) error { msg.InitTimes() - msg.SetUid(uGen.Get()) + msg.SetUid(GetUid()) // Increment topic's or user's SeqId err := adp.TopicUpdateOnMessage(msg.Topic, msg) if err != nil { @@ -571,7 +571,7 @@ func (MessagesObjMapper) DeleteList(topic string, delID int, forUser types.Uid, DelId: delID, DeletedFor: forUser.String(), SeqIdRanges: ranges} - toDel.SetUid(uGen.Get()) + toDel.SetUid(GetUid()) toDel.InitTimes() } From af21afb6b1d10e6bf527b360f6564707877fa4e3 Mon Sep 17 00:00:00 2001 From: or-else Date: Sat, 16 Nov 2019 10:13:56 +0300 Subject: [PATCH 11/13] fix for a deadlock in hub.join --- server/hub.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/hub.go b/server/hub.go index 05f19491b..0761350af 100644 --- a/server/hub.go +++ b/server/hub.go @@ -263,7 +263,10 @@ func (h *Hub) run() { // Check if 'sys' topic has migrated to this node. if h.topicGet("sys") == nil && !globals.cluster.isRemoteTopic("sys") { // Yes, 'sys' has migrated here. Initialize it. - h.join <- &sessionJoin{topic: "sys", internal: true, pkt: &ClientComMessage{topic: "sys"}} + // The h.join is unbuffered. We must call from another goroutine. Otherwise deadlock. + go func() { + h.join <- &sessionJoin{topic: "sys", internal: true, pkt: &ClientComMessage{topic: "sys"}} + }() } case hubdone := <-h.shutdown: From 31c79c8c91d6944b42ac1127eda4df65042345ff Mon Sep 17 00:00:00 2001 From: or-else Date: Sat, 16 Nov 2019 10:20:58 +0300 Subject: [PATCH 12/13] adding an explanation wny store-assigned messsage ID is not used --- server/db/mysql/adapter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/db/mysql/adapter.go b/server/db/mysql/adapter.go index 64aab0260..a8f4cd89e 100644 --- a/server/db/mysql/adapter.go +++ b/server/db/mysql/adapter.go @@ -1864,12 +1864,15 @@ func (a *adapter) FindTopics(req, opt []string) ([]t.Subscription, error) { // Messages func (a *adapter) MessageSave(msg *t.Message) error { + // store assignes message ID, but we don't use it. Message IDs are not used anywhere. + // Using a sequential ID provided by the database. res, err := a.db.Exec( "INSERT INTO messages(createdAt,updatedAt,seqid,topic,`from`,head,content) VALUES(?,?,?,?,?,?,?)", msg.CreatedAt, msg.UpdatedAt, msg.SeqId, msg.Topic, store.DecodeUid(t.ParseUid(msg.From)), msg.Head, toJSON(msg.Content)) if err == nil { id, _ := res.LastInsertId() + // Replacing ID given by store by ID given by the DB. msg.SetUid(t.Uid(id)) } return err From 17c694a447dd66d891f505b5185622210a96b8ca Mon Sep 17 00:00:00 2001 From: aforge Date: Sat, 16 Nov 2019 18:00:07 -0800 Subject: [PATCH 13/13] Set MutableContent and Alert field on the FCM Aps. Needed for NotificationServiceExtension to work. Also, enable sound on push notifications in iOS. --- server/push/fcm/push_fcm.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/server/push/fcm/push_fcm.go b/server/push/fcm/push_fcm.go index 3f9d853de..5a4d8c195 100644 --- a/server/push/fcm/push_fcm.go +++ b/server/push/fcm/push_fcm.go @@ -191,14 +191,26 @@ func sendNotifications(rcpt *push.Receipt, config *configType) { } else if d.Platform == "ios" { // iOS uses Badge to show the total unread message count. badge := rcpt.To[uid].Unread + // Need to duplicate these in APNS.Payload.Aps.Alert so + // iOS may call NotificationServiceExtension (if present). + title := "New message" + body := data["content"] msg.APNS = &fcm.APNSConfig{ Payload: &fcm.APNSPayload{ - Aps: &fcm.Aps{Badge: &badge}, + Aps: &fcm.Aps{ + Badge: &badge, + MutableContent: true, + Sound: "default", + Alert: &fcm.ApsAlert{ + Title: title, + Body: body, + }, + }, }, } msg.Notification = &fcm.Notification{ - Title: "New message", - Body: data["content"], + Title: title, + Body: body, } }