web: move and simplify guild caching

cmd/yagpdb/main.go

@@ -5,6 +5,7 @@ import (
 	"github.com/jonas747/yagpdb/common/featureflags"
 	"github.com/jonas747/yagpdb/common/prom"
 	"github.com/jonas747/yagpdb/common/run"
+	"github.com/jonas747/yagpdb/web/discorddata"
 
 	// Core yagpdb packages
 
@@ -51,6 +52,7 @@ func main() {
 
 	//BotSession.LogLevel = discordgo.LogInformational
 	paginatedmessages.RegisterPlugin()
+	discorddata.RegisterPlugin()
 
 	// Setup plugins
 	analytics.RegisterPlugin()

common/util.go

@@ -74,34 +74,6 @@ func SendTempMessage(session *discordgo.Session, duration time.Duration, cID int
 	DelayedMessageDelete(session, duration, cID, m.ID)
 }
 
-// GetGuildChannels returns the guilds channels either from cache or api
-func GetGuildChannels(guildID int64) (channels []*discordgo.Channel, err error) {
-	// Check cache first
-	err = GetCacheDataJson(KeyGuildChannels(guildID), &channels)
-	if err != nil {
-		channels, err = BotSession.GuildChannels(guildID)
-		if err == nil {
-			SetCacheDataJsonSimple(KeyGuildChannels(guildID), channels)
-		}
-	}
-
-	return
-}
-
-// GetGuild returns the guild from guildid either from cache or api
-func GetGuild(guildID int64) (guild *discordgo.Guild, err error) {
-	// Check cache first
-	err = GetCacheDataJson(KeyGuild(guildID), &guild)
-	if err != nil {
-		guild, err = BotSession.Guild(guildID)
-		if err == nil {
-			SetCacheDataJsonSimple(KeyGuild(guildID), guild)
-		}
-	}
-
-	return
-}
-
 func RandomAdjective() string {
 	return Adjectives[rand.Intn(len(Adjectives))]
 }

web/discorddata/bot.go

@@ -0,0 +1,47 @@
+package discorddata
+
+import (
+	"github.com/jonas747/yagpdb/bot"
+	"github.com/jonas747/yagpdb/bot/eventsystem"
+	"github.com/jonas747/yagpdb/common/pubsub"
+)
+
+type EvictData struct {
+	Keys []string `json:"keys"`
+}
+
+func init() {
+	pubsub.AddHandler("web_discorddata_evict", func(event *pubsub.Event) {
+		data := event.Data.(*EvictData)
+
+		for _, v := range data.Keys {
+			applicationCache.Delete(v)
+		}
+	}, EvictData{})
+}
+
+var _ bot.BotInitHandler = (*Plugin)(nil)
+
+func (p *Plugin) BotInit() {
+	eventsystem.AddHandlerAsyncLast(p, p.handleInvalidateGuildCache, eventsystem.EventGuildRoleCreate,
+		eventsystem.EventGuildRoleUpdate,
+		eventsystem.EventGuildRoleDelete,
+		eventsystem.EventChannelCreate,
+		eventsystem.EventChannelUpdate,
+		eventsystem.EventChannelDelete)
+}
+
+func (p *Plugin) handleInvalidateGuildCache(evt *eventsystem.EventData) (retry bool, err error) {
+	PubEvictGuild(evt.GS.ID)
+	return false, nil
+}
+
+func pubEvictCache(keys ...string) {
+	pubsub.Publish("web_discorddata_evict", -1, EvictData{
+		Keys: keys,
+	})
+}
+
+func PubEvictGuild(guildID int64) {
+	pubEvictCache(keyFullGuild(guildID))
+}

web/discorddata/discord_data.go

@@ -1,24 +1,41 @@
 package discorddata
 
 import (
+	"sort"
+	"strconv"
 	"time"
 
 	"emperror.dev/errors"
 	"github.com/jonas747/discordgo"
+	"github.com/jonas747/dutil"
+	"github.com/jonas747/yagpdb/bot/botrest"
+	"github.com/jonas747/yagpdb/common"
 	"github.com/karlseguin/ccache"
 	"golang.org/x/oauth2"
 )
 
+type Plugin struct{}
+
+func (p *Plugin) PluginInfo() *common.PluginInfo {
+	return &common.PluginInfo{
+		Name:     "web_discorddata",
+		SysName:  "web_discorddata",
+		Category: common.PluginCategoryMisc,
+	}
+}
+
+var logger = common.GetPluginLogger(&Plugin{})
+
+func RegisterPlugin() {
+	common.RegisterPlugin(&Plugin{})
+}
+
 var applicationCache = ccache.New(ccache.Configure().MaxSize(10000).ItemsToPrune(100))
 
 func keySession(raw string) string {
 	return "discord_session:" + raw
 }
 
-func keyUserInfo(token string) string {
-	return "user_info_token:" + token
-}
-
 func GetSession(raw string, tokenDecoder func(string) (*oauth2.Token, error)) (*discordgo.Session, error) {
 	result, err := applicationCache.Fetch(keySession(raw), time.Minute*10, func() (interface{}, error) {
 		decoded, err := tokenDecoder(raw)
@@ -40,6 +57,10 @@ func GetSession(raw string, tokenDecoder func(string) (*oauth2.Token, error)) (*
 	return result.Value().(*discordgo.Session), nil
 }
 
+func keyUserInfo(token string) string {
+	return "user_info_token:" + token
+}
+
 func GetUserInfo(token string, session *discordgo.Session) (*discordgo.User, error) {
 	result, err := applicationCache.Fetch(keyUserInfo(token), time.Minute*10, func() (interface{}, error) {
 		user, err := session.UserMe()
@@ -56,3 +77,44 @@ func GetUserInfo(token string, session *discordgo.Session) (*discordgo.User, err
 
 	return result.Value().(*discordgo.User), nil
 }
+
+func keyFullGuild(guildID int64) string {
+	return "full_guild:" + strconv.FormatInt(guildID, 10)
+}
+
+// GetFullGuild returns the guild from either:
+// 1. Application cache
+// 2. Botrest
+// 3. Discord api
+//
+// It will will also make sure channels are included in the event we fall back to the discord API
+func GetFullGuild(guildID int64) (*discordgo.Guild, error) {
+	result, err := applicationCache.Fetch(keyFullGuild(guildID), time.Minute*10, func() (interface{}, error) {
+		guild, err := botrest.GetGuild(guildID)
+		if err != nil {
+			// fall back to discord API
+			guild, err = common.BotSession.Guild(guildID)
+			if err != nil {
+				return nil, err
+			}
+
+			// we also need to include channels as they're not included in the guild response
+			channels, err := common.BotSession.GuildChannels(guildID)
+			if err != nil {
+				return nil, err
+			}
+
+			guild.Channels = channels
+		}
+
+		sort.Sort(dutil.Channels(guild.Channels))
+
+		return guild, nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return result.Value().(*discordgo.Guild), nil
+}

web/middleware.go

@@ -242,30 +242,11 @@ func UserInfoMiddleware(inner http.Handler) http.Handler {
 	return http.HandlerFunc(mw)
 }
 
-// setFullGuild is a fallback in case a userguild is not available, could be the case if a bot admin is accesing a server they're not part of
-func setFullGuild(ctx context.Context, guildID int64) (context.Context, error) {
-	fullGuild, err := common.GetGuild(guildID)
+func getGuild(ctx context.Context, guildID int64) (*discordgo.Guild, error) {
+	guild, err := discorddata.GetFullGuild(guildID)
 	if err != nil {
-		CtxLogger(ctx).WithError(err).Error("Failed retrieving guild")
-		return ctx, err
-	}
-
-	entry := CtxLogger(ctx).WithField("g", guildID)
-	ctx = context.WithValue(ctx, common.ContextKeyLogger, entry)
-	ctx = SetContextTemplateData(ctx, map[string]interface{}{"ActiveGuild": fullGuild})
-	return context.WithValue(ctx, common.ContextKeyCurrentGuild, fullGuild), nil
-}
-
-func getGuild(guildID int64, ctx context.Context) (*discordgo.Guild, error) {
-	guild, err := botrest.GetGuild(guildID)
-	if err != nil {
-		CtxLogger(ctx).WithError(err).Warn("failed getting guild from bot, querying discord api")
-
-		guild, err = common.BotSession.Guild(guildID)
-		if err != nil {
-			CtxLogger(ctx).WithError(err).Warn("failed getting guild from discord fallback, nothing more we can do...")
-			return nil, err
-		}
+		CtxLogger(ctx).WithError(err).Warn("failed getting guild from discord fallback, nothing more we can do...")
+		return nil, err
 	}
 
 	return guild, nil
@@ -286,7 +267,7 @@ func ActiveServerMW(inner http.Handler) http.Handler {
 			return
 		}
 
-		guild, err := getGuild(guildID, ctx)
+		guild, err := getGuild(ctx, guildID)
 		if err != nil {
 			return
 		}
@@ -361,27 +342,6 @@ func RequireServerAdminMiddleware(inner http.Handler) http.Handler {
 // RequireGuildChannelsMiddleware ensures that the channels are available for the guild were on during this request, and yes this has to be done seperately cause discord
 func RequireGuildChannelsMiddleware(inner http.Handler) http.Handler {
 	mw := func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		guild := ctx.Value(common.ContextKeyCurrentGuild).(*discordgo.Guild)
-
-		if len(guild.Channels) > 0 {
-			// channels already available
-			sort.Sort(dutil.Channels(guild.Channels))
-			inner.ServeHTTP(w, r)
-			return
-		}
-
-		channels, err := common.GetGuildChannels(guild.ID)
-		if err != nil {
-			CtxLogger(ctx).WithError(err).Error("Failed retrieving channels")
-			http.Redirect(w, r, "/?err=retrievingchannels", http.StatusTemporaryRedirect)
-			return
-		}
-
-		// Sort them
-		sort.Sort(dutil.Channels(channels))
-		guild.Channels = channels
-
 		inner.ServeHTTP(w, r)
 	}
 	return http.HandlerFunc(mw)