From 8608a4970c4505231572838ff42ef9736d9accc6 Mon Sep 17 00:00:00 2001 From: gugray Date: Mon, 1 Apr 2024 16:41:40 +0200 Subject: [PATCH] #45 more post metrics; post purge tweaks+config --- src/server/dal/repo.go | 16 ++++++ src/server/logic/feed_follower.go | 51 +++++++++++++------ src/server/logic/metrics.go | 49 +++++++++++++----- src/server/shared/config.go | 28 +++++----- .../test/apub_entities_serialization_test.go | 3 ++ src/server/test/feed_follower_purge_test.go | 2 + src/server/test/mocks/mock_metrics.go | 24 +++++++++ src/server/test/mocks/mock_repo.go | 15 ++++++ src/server/test/scaffolding_misc.go | 2 + 9 files changed, 149 insertions(+), 41 deletions(-) diff --git a/src/server/dal/repo.go b/src/server/dal/repo.go index a4f71a6..ce30526 100644 --- a/src/server/dal/repo.go +++ b/src/server/dal/repo.go @@ -31,6 +31,7 @@ type IRepo interface { AddToot(accountId int, toot *Toot) error GetToot(statusId string) (*Toot, error) GetPostCount(user string) (uint, error) + GetTotalPostCount() (uint, error) GetPostsPage(accountId int, offset, limit int) ([]*FeedPost, error) GetPostsExtract(accountId int) ([]*FeedPost, error) GetFeedLastUpdated(accountId int) (time.Time, error) @@ -441,6 +442,20 @@ func (repo *Repo) GetPostCount(user string) (uint, error) { return uint(count), nil } +func (repo *Repo) GetTotalPostCount() (uint, error) { + + repo.muDb.RLock() + defer repo.muDb.RUnlock() + + row := repo.db.QueryRow(`SELECT COUNT(*) FROM feed_posts`) + var err error + var count int + if err = row.Scan(&count); err != nil { + return 0, err + } + return uint(count), nil +} + func (repo *Repo) GetPostsPage(accountId int, offset, limit int) ([]*FeedPost, error) { repo.muDb.RLock() @@ -795,6 +810,7 @@ func (repo *Repo) PurgePostsAndToots(accountId int, postGuidHashes []int64) erro // Release lock periodically to let readers proceed if ((i + 1) % 10) == 0 { repo.muDb.Unlock() + time.Sleep(time.Duration(repo.cfg.PostDeleteBatchWaitSec) * time.Second) repo.muDb.Lock() } } diff --git a/src/server/logic/feed_follower.go b/src/server/logic/feed_follower.go index 2c6aaf4..20c4948 100644 --- a/src/server/logic/feed_follower.go +++ b/src/server/logic/feed_follower.go @@ -23,7 +23,7 @@ import ( //go:generate mockgen --build_flags=--mod=mod -destination ../test/mocks/mock_feed_follower.go -package mocks rss_parrot/logic IFeedFollower const feedCheckLoopIdleWakeSec = 60 -const purgeWaitSec = 20 +const postCountUpdateSecs = 60 type FeedStatus int32 @@ -55,17 +55,18 @@ type SiteInfo struct { } type feedFollower struct { - cfg *shared.Config - logger shared.ILogger - userAgent shared.IUserAgent - repo dal.IRepo - blockedFeeds IBlockedFeeds - messenger IMessenger - txt texts.ITexts - keyStore IKeyStore - metrics IMetrics - muDeleting sync.Mutex - isDeleting bool + cfg *shared.Config + logger shared.ILogger + userAgent shared.IUserAgent + repo dal.IRepo + blockedFeeds IBlockedFeeds + messenger IMessenger + txt texts.ITexts + keyStore IKeyStore + metrics IMetrics + lastCheckedPostCount time.Time + muDeleting sync.Mutex + isDeleting bool } func NewFeedFollower( @@ -94,6 +95,7 @@ func NewFeedFollower( } ff.updateDBSizeMetric() + ff.updateTotalPostsMetric() go ff.feedCheckLoop() return &ff @@ -647,8 +649,8 @@ func (ff *feedFollower) PurgeOldPosts(acct *dal.Account, minCount, minAgeDays in ff.muDeleting.Unlock() } defer signalDone() - if purgeWaitSec > 0 { - time.Sleep(purgeWaitSec * time.Second) + if ff.cfg.PurgeWaitSec > 0 { + time.Sleep(time.Duration(ff.cfg.PurgeWaitSec) * time.Second) } var err error @@ -686,6 +688,7 @@ func (ff *feedFollower) PurgeOldPosts(acct *dal.Account, minCount, minAgeDays in if err = ff.repo.PurgePostsAndToots(acct.Id, hashesToDel); err != nil { return err } + ff.metrics.PostsDeleted(len(hashesToDel)) return nil } @@ -723,8 +726,8 @@ func (ff *feedFollower) purgeUnfollowedAccount(acct *dal.Account) { ff.logger.Errorf("Failed to brute-delete account: %s: %v", acct.Handle, err) return } - if purgeWaitSec > 0 { - time.Sleep(purgeWaitSec * time.Second) + if ff.cfg.PurgeWaitSec > 0 { + time.Sleep(time.Duration(ff.cfg.PurgeWaitSec) * time.Second) } } @@ -745,6 +748,21 @@ func (ff *feedFollower) updateDBSizeMetric() { ff.metrics.DbFileSize(fi.Size()) } +func (ff *feedFollower) updateTotalPostsMetric() { + now := time.Now() + if now.Sub(ff.lastCheckedPostCount).Seconds() < postCountUpdateSecs { + return + } + ff.lastCheckedPostCount = now + ff.logger.Info("Updating total post count metric") + if count, err := ff.repo.GetTotalPostCount(); err != nil { + ff.logger.Errorf("Error getting total post count: %v", err) + return + } else { + ff.metrics.TotalPosts(int(count)) + } +} + func (ff *feedFollower) feedCheckLoop() { for { // This is why we're here @@ -755,6 +773,7 @@ func (ff *feedFollower) feedCheckLoop() { // Rather a little ugliness here, then all that boilerplate // And we're already also setting the "feed-followers" metrics in this module ff.updateDBSizeMetric() + ff.updateTotalPostsMetric() } } diff --git a/src/server/logic/metrics.go b/src/server/logic/metrics.go index e330aeb..19172d5 100644 --- a/src/server/logic/metrics.go +++ b/src/server/logic/metrics.go @@ -15,6 +15,8 @@ type IMetrics interface { FeedRequested(label string) FeedUpdated() NewPostSaved() + PostsDeleted(count int) + TotalPosts(count int) FeedTootSent() ServiceStarted() TotalFollowers(count int) @@ -33,11 +35,13 @@ type metrics struct { apubRequestsIn *prometheus.HistogramVec apubRequestsOut *prometheus.HistogramVec feedsRequested *prometheus.CounterVec + postFlow *prometheus.CounterVec feedsUpdated prometheus.Counter newPostsSaved prometheus.Counter feedTootsSent prometheus.Counter serviceStarted prometheus.Counter totalFollowers prometheus.Gauge + totalPosts prometheus.Gauge tootQueueLength prometheus.Gauge checkableFeedCount prometheus.Gauge dbFileSize prometheus.Gauge @@ -52,73 +56,85 @@ func NewMetrics(cfg *shared.Config) IMetrics { Name: "web_requests_in_duration", Help: "Duration in seconds of Web requests served.", }, []string{"label"}) - prometheus.Register(res.webRequestsIn) + _ = prometheus.Register(res.webRequestsIn) res.apubRequestsIn = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "apub_requests_in_duration", Help: "Duration in seconds of ActivityPub requests served.", }, []string{"label"}) - prometheus.Register(res.apubRequestsIn) + _ = prometheus.Register(res.apubRequestsIn) res.apubRequestsOut = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "apub_requests_out_duration", Help: "Duration in seconds of ActivityPub requests made.", }, []string{"label"}) - prometheus.Register(res.apubRequestsOut) + _ = prometheus.Register(res.apubRequestsOut) res.feedsRequested = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "feeds_requested", Help: "Number of feeds requested", }, []string{"label"}) - prometheus.Register(res.feedsRequested) + _ = prometheus.Register(res.feedsRequested) + + res.postFlow = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "post_flow", + Help: "Posts added and deleted", + }, []string{"label"}) + _ = prometheus.Register(res.postFlow) res.feedsUpdated = prometheus.NewCounter(prometheus.CounterOpts{ Name: "feeds_updated", Help: "Number of feeds updated", }) - prometheus.Register(res.feedsUpdated) + _ = prometheus.Register(res.feedsUpdated) res.newPostsSaved = prometheus.NewCounter(prometheus.CounterOpts{ Name: "new_posts_saved", Help: "Number of new posts saved", }) - prometheus.Register(res.newPostsSaved) + _ = prometheus.Register(res.newPostsSaved) res.feedTootsSent = prometheus.NewCounter(prometheus.CounterOpts{ Name: "feed_toots_sent", Help: "Number of toots sent about new feed posts", }) - prometheus.Register(res.feedTootsSent) + _ = prometheus.Register(res.feedTootsSent) res.serviceStarted = prometheus.NewCounter(prometheus.CounterOpts{ Name: "service_started", Help: "Service has started up", }) - prometheus.Register(res.serviceStarted) + _ = prometheus.Register(res.serviceStarted) res.totalFollowers = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "total_follower_count", Help: "Total follower count of parrot accounts", }) - prometheus.Register(res.totalFollowers) + _ = prometheus.Register(res.totalFollowers) + + res.totalPosts = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "total_post_count", + Help: "Total number of posts stored", + }) + _ = prometheus.Register(res.totalPosts) res.tootQueueLength = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "toot_queue_length", Help: "Items in toot queue", }) - prometheus.Register(res.tootQueueLength) + _ = prometheus.Register(res.tootQueueLength) res.checkableFeedCount = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "checkable_feed_count", Help: "Number of feeds waiting to be checked", }) - prometheus.Register(res.checkableFeedCount) + _ = prometheus.Register(res.checkableFeedCount) res.dbFileSize = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "db_file_size", Help: "SQLite database file size", }) - prometheus.Register(res.dbFileSize) + _ = prometheus.Register(res.dbFileSize) return &res } @@ -165,6 +181,11 @@ func (m *metrics) FeedTootSent() { func (m *metrics) NewPostSaved() { m.newPostsSaved.Add(1) + m.postFlow.WithLabelValues("saved").Add(1) +} + +func (m *metrics) PostsDeleted(count int) { + m.postFlow.WithLabelValues("purged").Add(float64(count)) } func (m *metrics) ServiceStarted() { @@ -175,6 +196,10 @@ func (m *metrics) TotalFollowers(count int) { m.totalFollowers.Set(float64(count)) } +func (m *metrics) TotalPosts(count int) { + m.totalPosts.Set(float64(count)) +} + func (m *metrics) CheckableFeedCount(count int) { m.checkableFeedCount.Set(float64(count)) } diff --git a/src/server/shared/config.go b/src/server/shared/config.go index 3eed38d..f4b9521 100644 --- a/src/server/shared/config.go +++ b/src/server/shared/config.go @@ -16,19 +16,21 @@ const ( ) type Config struct { - Secrets Secrets `json:"-"` - LogFile string `json:"log_file"` - LogLevel string `json:"log_level"` - ServicePort uint `json:"service_port"` - Host string `json:"host"` - DbFile string `json:"db_file"` - BlockedFeedsFile string `json:"blocked_feeds_file"` - CachePageTemplates bool `json:"cache_page_templates"` - UpdateSchedule UpdateSchedule `json:"update_schedule"` - PostsMinCountKept int `json:"posts_min_count_kept"` - PostsMinDaysKept int `json:"posts_min_days_kept"` - FallbackProfilePic string `json:"fallback_profile_pic"` - Birb *UserInfo `json:"birb"` + Secrets Secrets `json:"-"` + LogFile string `json:"log_file"` + LogLevel string `json:"log_level"` + ServicePort uint `json:"service_port"` + Host string `json:"host"` + DbFile string `json:"db_file"` + BlockedFeedsFile string `json:"blocked_feeds_file"` + CachePageTemplates bool `json:"cache_page_templates"` + UpdateSchedule UpdateSchedule `json:"update_schedule"` + PostsMinCountKept int `json:"posts_min_count_kept"` + PostsMinDaysKept int `json:"posts_min_days_kept"` + PurgeWaitSec int `json:"purge_wait_sec"` + PostDeleteBatchWaitSec float64 `json:"post_delete_batch_wait_sec"` + FallbackProfilePic string `json:"fallback_profile_pic"` + Birb *UserInfo `json:"birb"` } type UpdateSchedule struct { diff --git a/src/server/test/apub_entities_serialization_test.go b/src/server/test/apub_entities_serialization_test.go index bdd8b88..c5d1e2f 100644 --- a/src/server/test/apub_entities_serialization_test.go +++ b/src/server/test/apub_entities_serialization_test.go @@ -46,3 +46,6 @@ func Test_Deserialize_Note(t *testing.T) { assert.Equal(t, "@birb@rss-parrot.zydeo.net", (*note.Tag)[0].Name) assert.Equal(t, "Mention", (*note.Tag)[0].Type) } + +//func Test_Foo(t *testing.T) { +//} diff --git a/src/server/test/feed_follower_purge_test.go b/src/server/test/feed_follower_purge_test.go index fbb1340..72738ac 100644 --- a/src/server/test/feed_follower_purge_test.go +++ b/src/server/test/feed_follower_purge_test.go @@ -46,6 +46,8 @@ func setupFeedFollowerTest(t *testing.T) (*gomock.Controller, *feedFollowerHarne setupDummyLogger(h.mockLogger) setupDummyMetrics(h.mockMetrics) + h.mockRepo.EXPECT().GetTotalPostCount().Return(uint(0), nil).AnyTimes() + ff := logic.NewFeedFollower(h.cfg, h.mockLogger, h.mockUserAgent, h.mockRepo, h.mockBlockedFeeds, h.mockMessenger, h.mockTexts, h.mockKeyStore, h.mockMetrics) diff --git a/src/server/test/mocks/mock_metrics.go b/src/server/test/mocks/mock_metrics.go index 182bb41..83cf125 100644 --- a/src/server/test/mocks/mock_metrics.go +++ b/src/server/test/mocks/mock_metrics.go @@ -111,6 +111,18 @@ func (mr *MockIMetricsMockRecorder) NewPostSaved() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewPostSaved", reflect.TypeOf((*MockIMetrics)(nil).NewPostSaved)) } +// PostsDeleted mocks base method. +func (m *MockIMetrics) PostsDeleted(arg0 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "PostsDeleted", arg0) +} + +// PostsDeleted indicates an expected call of PostsDeleted. +func (mr *MockIMetricsMockRecorder) PostsDeleted(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostsDeleted", reflect.TypeOf((*MockIMetrics)(nil).PostsDeleted), arg0) +} + // ServiceStarted mocks base method. func (m *MockIMetrics) ServiceStarted() { m.ctrl.T.Helper() @@ -188,3 +200,15 @@ func (mr *MockIMetricsMockRecorder) TotalFollowers(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TotalFollowers", reflect.TypeOf((*MockIMetrics)(nil).TotalFollowers), arg0) } + +// TotalPosts mocks base method. +func (m *MockIMetrics) TotalPosts(arg0 int) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "TotalPosts", arg0) +} + +// TotalPosts indicates an expected call of TotalPosts. +func (mr *MockIMetricsMockRecorder) TotalPosts(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TotalPosts", reflect.TypeOf((*MockIMetrics)(nil).TotalPosts), arg0) +} diff --git a/src/server/test/mocks/mock_repo.go b/src/server/test/mocks/mock_repo.go index b09a1f0..c148ecf 100644 --- a/src/server/test/mocks/mock_repo.go +++ b/src/server/test/mocks/mock_repo.go @@ -396,6 +396,21 @@ func (mr *MockIRepoMockRecorder) GetTootQueueItems(arg0, arg1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTootQueueItems", reflect.TypeOf((*MockIRepo)(nil).GetTootQueueItems), arg0, arg1) } +// GetTotalPostCount mocks base method. +func (m *MockIRepo) GetTotalPostCount() (uint, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTotalPostCount") + ret0, _ := ret[0].(uint) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTotalPostCount indicates an expected call of GetTotalPostCount. +func (mr *MockIRepoMockRecorder) GetTotalPostCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTotalPostCount", reflect.TypeOf((*MockIRepo)(nil).GetTotalPostCount)) +} + // InitUpdateDb mocks base method. func (m *MockIRepo) InitUpdateDb() { m.ctrl.T.Helper() diff --git a/src/server/test/scaffolding_misc.go b/src/server/test/scaffolding_misc.go index 631bb1b..7a24378 100644 --- a/src/server/test/scaffolding_misc.go +++ b/src/server/test/scaffolding_misc.go @@ -58,6 +58,8 @@ func fakeTextWithVals(id string, vals map[string]string) string { func setupDummyMetrics(mockMetrics *mocks.MockIMetrics) { mockMetrics.EXPECT().TotalFollowers(gomock.Any()).AnyTimes() + mockMetrics.EXPECT().TotalPosts(gomock.Any()).AnyTimes() + mockMetrics.EXPECT().PostsDeleted(gomock.Any()).AnyTimes() mockMetrics.EXPECT().CheckableFeedCount(gomock.Any()).AnyTimes() }