#45 more post metrics; post purge tweaks+config

src/server/dal/repo.go

@@ -31,6 +31,7 @@ type IRepo interface {
 	AddToot(accountId int, toot *Toot) error
 	GetToot(statusId string) (*Toot, error)
 	GetPostCount(user string) (uint, error)
+	GetTotalPostCount() (uint, error)
 	GetPostsPage(accountId int, offset, limit int) ([]*FeedPost, error)
 	GetPostsExtract(accountId int) ([]*FeedPost, error)
 	GetFeedLastUpdated(accountId int) (time.Time, error)
@@ -441,6 +442,20 @@ func (repo *Repo) GetPostCount(user string) (uint, error) {
 	return uint(count), nil
 }
 
+func (repo *Repo) GetTotalPostCount() (uint, error) {
+
+	repo.muDb.RLock()
+	defer repo.muDb.RUnlock()
+
+	row := repo.db.QueryRow(`SELECT COUNT(*) FROM feed_posts`)
+	var err error
+	var count int
+	if err = row.Scan(&count); err != nil {
+		return 0, err
+	}
+	return uint(count), nil
+}
+
 func (repo *Repo) GetPostsPage(accountId int, offset, limit int) ([]*FeedPost, error) {
 
 	repo.muDb.RLock()
@@ -795,6 +810,7 @@ func (repo *Repo) PurgePostsAndToots(accountId int, postGuidHashes []int64) erro
 		// Release lock periodically to let readers proceed
 		if ((i + 1) % 10) == 0 {
 			repo.muDb.Unlock()
+			time.Sleep(time.Duration(repo.cfg.PostDeleteBatchWaitSec) * time.Second)
 			repo.muDb.Lock()
 		}
 	}

src/server/logic/feed_follower.go

@@ -23,7 +23,7 @@ import (
 //go:generate mockgen --build_flags=--mod=mod -destination ../test/mocks/mock_feed_follower.go -package mocks rss_parrot/logic IFeedFollower
 
 const feedCheckLoopIdleWakeSec = 60
-const purgeWaitSec = 20
+const postCountUpdateSecs = 60
 
 type FeedStatus int32
 
@@ -55,17 +55,18 @@ type SiteInfo struct {
 }
 
 type feedFollower struct {
-	cfg          *shared.Config
-	logger       shared.ILogger
-	userAgent    shared.IUserAgent
-	repo         dal.IRepo
-	blockedFeeds IBlockedFeeds
-	messenger    IMessenger
-	txt          texts.ITexts
-	keyStore     IKeyStore
-	metrics      IMetrics
-	muDeleting   sync.Mutex
-	isDeleting   bool
+	cfg                  *shared.Config
+	logger               shared.ILogger
+	userAgent            shared.IUserAgent
+	repo                 dal.IRepo
+	blockedFeeds         IBlockedFeeds
+	messenger            IMessenger
+	txt                  texts.ITexts
+	keyStore             IKeyStore
+	metrics              IMetrics
+	lastCheckedPostCount time.Time
+	muDeleting           sync.Mutex
+	isDeleting           bool
 }
 
 func NewFeedFollower(
@@ -94,6 +95,7 @@ func NewFeedFollower(
 	}
 
 	ff.updateDBSizeMetric()
+	ff.updateTotalPostsMetric()
 	go ff.feedCheckLoop()
 
 	return &ff
@@ -647,8 +649,8 @@ func (ff *feedFollower) PurgeOldPosts(acct *dal.Account, minCount, minAgeDays in
 		ff.muDeleting.Unlock()
 	}
 	defer signalDone()
-	if purgeWaitSec > 0 {
-		time.Sleep(purgeWaitSec * time.Second)
+	if ff.cfg.PurgeWaitSec > 0 {
+		time.Sleep(time.Duration(ff.cfg.PurgeWaitSec) * time.Second)
 	}
 
 	var err error
@@ -686,6 +688,7 @@ func (ff *feedFollower) PurgeOldPosts(acct *dal.Account, minCount, minAgeDays in
 	if err = ff.repo.PurgePostsAndToots(acct.Id, hashesToDel); err != nil {
 		return err
 	}
+	ff.metrics.PostsDeleted(len(hashesToDel))
 	return nil
 }
 
@@ -723,8 +726,8 @@ func (ff *feedFollower) purgeUnfollowedAccount(acct *dal.Account) {
 		ff.logger.Errorf("Failed to brute-delete account: %s: %v", acct.Handle, err)
 		return
 	}
-	if purgeWaitSec > 0 {
-		time.Sleep(purgeWaitSec * time.Second)
+	if ff.cfg.PurgeWaitSec > 0 {
+		time.Sleep(time.Duration(ff.cfg.PurgeWaitSec) * time.Second)
 	}
 }
 
@@ -745,6 +748,21 @@ func (ff *feedFollower) updateDBSizeMetric() {
 	ff.metrics.DbFileSize(fi.Size())
 }
 
+func (ff *feedFollower) updateTotalPostsMetric() {
+	now := time.Now()
+	if now.Sub(ff.lastCheckedPostCount).Seconds() < postCountUpdateSecs {
+		return
+	}
+	ff.lastCheckedPostCount = now
+	ff.logger.Info("Updating total post count metric")
+	if count, err := ff.repo.GetTotalPostCount(); err != nil {
+		ff.logger.Errorf("Error getting total post count: %v", err)
+		return
+	} else {
+		ff.metrics.TotalPosts(int(count))
+	}
+}
+
 func (ff *feedFollower) feedCheckLoop() {
 	for {
 		// This is why we're here
@@ -755,6 +773,7 @@ func (ff *feedFollower) feedCheckLoop() {
 		// Rather a little ugliness here, then all that boilerplate
 		// And we're already also setting the "feed-followers" metrics in this module
 		ff.updateDBSizeMetric()
+		ff.updateTotalPostsMetric()
 	}
 }
 

src/server/logic/metrics.go

@@ -15,6 +15,8 @@ type IMetrics interface {
 	FeedRequested(label string)
 	FeedUpdated()
 	NewPostSaved()
+	PostsDeleted(count int)
+	TotalPosts(count int)
 	FeedTootSent()
 	ServiceStarted()
 	TotalFollowers(count int)
@@ -33,11 +35,13 @@ type metrics struct {
 	apubRequestsIn     *prometheus.HistogramVec
 	apubRequestsOut    *prometheus.HistogramVec
 	feedsRequested     *prometheus.CounterVec
+	postFlow           *prometheus.CounterVec
 	feedsUpdated       prometheus.Counter
 	newPostsSaved      prometheus.Counter
 	feedTootsSent      prometheus.Counter
 	serviceStarted     prometheus.Counter
 	totalFollowers     prometheus.Gauge
+	totalPosts         prometheus.Gauge
 	tootQueueLength    prometheus.Gauge
 	checkableFeedCount prometheus.Gauge
 	dbFileSize         prometheus.Gauge
@@ -52,73 +56,85 @@ func NewMetrics(cfg *shared.Config) IMetrics {
 		Name: "web_requests_in_duration",
 		Help: "Duration in seconds of Web requests served.",
 	}, []string{"label"})
-	prometheus.Register(res.webRequestsIn)
+	_ = prometheus.Register(res.webRequestsIn)
 
 	res.apubRequestsIn = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Name: "apub_requests_in_duration",
 		Help: "Duration in seconds of ActivityPub requests served.",
 	}, []string{"label"})
-	prometheus.Register(res.apubRequestsIn)
+	_ = prometheus.Register(res.apubRequestsIn)
 
 	res.apubRequestsOut = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Name: "apub_requests_out_duration",
 		Help: "Duration in seconds of ActivityPub requests made.",
 	}, []string{"label"})
-	prometheus.Register(res.apubRequestsOut)
+	_ = prometheus.Register(res.apubRequestsOut)
 
 	res.feedsRequested = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: "feeds_requested",
 		Help: "Number of feeds requested",
 	}, []string{"label"})
-	prometheus.Register(res.feedsRequested)
+	_ = prometheus.Register(res.feedsRequested)
+
+	res.postFlow = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "post_flow",
+		Help: "Posts added and deleted",
+	}, []string{"label"})
+	_ = prometheus.Register(res.postFlow)
 
 	res.feedsUpdated = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "feeds_updated",
 		Help: "Number of feeds updated",
 	})
-	prometheus.Register(res.feedsUpdated)
+	_ = prometheus.Register(res.feedsUpdated)
 
 	res.newPostsSaved = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "new_posts_saved",
 		Help: "Number of new posts saved",
 	})
-	prometheus.Register(res.newPostsSaved)
+	_ = prometheus.Register(res.newPostsSaved)
 
 	res.feedTootsSent = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "feed_toots_sent",
 		Help: "Number of toots sent about new feed posts",
 	})
-	prometheus.Register(res.feedTootsSent)
+	_ = prometheus.Register(res.feedTootsSent)
 
 	res.serviceStarted = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "service_started",
 		Help: "Service has started up",
 	})
-	prometheus.Register(res.serviceStarted)
+	_ = prometheus.Register(res.serviceStarted)
 
 	res.totalFollowers = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "total_follower_count",
 		Help: "Total follower count of parrot accounts",
 	})
-	prometheus.Register(res.totalFollowers)
+	_ = prometheus.Register(res.totalFollowers)
+
+	res.totalPosts = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "total_post_count",
+		Help: "Total number of posts stored",
+	})
+	_ = prometheus.Register(res.totalPosts)
 
 	res.tootQueueLength = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "toot_queue_length",
 		Help: "Items in toot queue",
 	})
-	prometheus.Register(res.tootQueueLength)
+	_ = prometheus.Register(res.tootQueueLength)
 
 	res.checkableFeedCount = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "checkable_feed_count",
 		Help: "Number of feeds waiting to be checked",
 	})
-	prometheus.Register(res.checkableFeedCount)
+	_ = prometheus.Register(res.checkableFeedCount)
 
 	res.dbFileSize = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "db_file_size",
 		Help: "SQLite database file size",
 	})
-	prometheus.Register(res.dbFileSize)
+	_ = prometheus.Register(res.dbFileSize)
 
 	return &res
 }
@@ -165,6 +181,11 @@ func (m *metrics) FeedTootSent() {
 
 func (m *metrics) NewPostSaved() {
 	m.newPostsSaved.Add(1)
+	m.postFlow.WithLabelValues("saved").Add(1)
+}
+
+func (m *metrics) PostsDeleted(count int) {
+	m.postFlow.WithLabelValues("purged").Add(float64(count))
 }
 
 func (m *metrics) ServiceStarted() {
@@ -175,6 +196,10 @@ func (m *metrics) TotalFollowers(count int) {
 	m.totalFollowers.Set(float64(count))
 }
 
+func (m *metrics) TotalPosts(count int) {
+	m.totalPosts.Set(float64(count))
+}
+
 func (m *metrics) CheckableFeedCount(count int) {
 	m.checkableFeedCount.Set(float64(count))
 }

src/server/shared/config.go

@@ -16,19 +16,21 @@ const (
 )
 
 type Config struct {
-	Secrets            Secrets        `json:"-"`
-	LogFile            string         `json:"log_file"`
-	LogLevel           string         `json:"log_level"`
-	ServicePort        uint           `json:"service_port"`
-	Host               string         `json:"host"`
-	DbFile             string         `json:"db_file"`
-	BlockedFeedsFile   string         `json:"blocked_feeds_file"`
-	CachePageTemplates bool           `json:"cache_page_templates"`
-	UpdateSchedule     UpdateSchedule `json:"update_schedule"`
-	PostsMinCountKept  int            `json:"posts_min_count_kept"`
-	PostsMinDaysKept   int            `json:"posts_min_days_kept"`
-	FallbackProfilePic string         `json:"fallback_profile_pic"`
-	Birb               *UserInfo      `json:"birb"`
+	Secrets                Secrets        `json:"-"`
+	LogFile                string         `json:"log_file"`
+	LogLevel               string         `json:"log_level"`
+	ServicePort            uint           `json:"service_port"`
+	Host                   string         `json:"host"`
+	DbFile                 string         `json:"db_file"`
+	BlockedFeedsFile       string         `json:"blocked_feeds_file"`
+	CachePageTemplates     bool           `json:"cache_page_templates"`
+	UpdateSchedule         UpdateSchedule `json:"update_schedule"`
+	PostsMinCountKept      int            `json:"posts_min_count_kept"`
+	PostsMinDaysKept       int            `json:"posts_min_days_kept"`
+	PurgeWaitSec           int            `json:"purge_wait_sec"`
+	PostDeleteBatchWaitSec float64        `json:"post_delete_batch_wait_sec"`
+	FallbackProfilePic     string         `json:"fallback_profile_pic"`
+	Birb                   *UserInfo      `json:"birb"`
 }
 
 type UpdateSchedule struct {

src/server/test/apub_entities_serialization_test.go

@@ -46,3 +46,6 @@ func Test_Deserialize_Note(t *testing.T) {
 	assert.Equal(t, "@birb@rss-parrot.zydeo.net", (*note.Tag)[0].Name)
 	assert.Equal(t, "Mention", (*note.Tag)[0].Type)
 }
+
+//func Test_Foo(t *testing.T) {
+//}

src/server/test/feed_follower_purge_test.go

@@ -46,6 +46,8 @@ func setupFeedFollowerTest(t *testing.T) (*gomock.Controller, *feedFollowerHarne
 	setupDummyLogger(h.mockLogger)
 	setupDummyMetrics(h.mockMetrics)
 
+	h.mockRepo.EXPECT().GetTotalPostCount().Return(uint(0), nil).AnyTimes()
+
 	ff := logic.NewFeedFollower(h.cfg, h.mockLogger, h.mockUserAgent, h.mockRepo,
 		h.mockBlockedFeeds, h.mockMessenger, h.mockTexts, h.mockKeyStore, h.mockMetrics)