Support Last-Modified header

Signed-off-by: AKP <tom@tdpain.net>

walrss/internal/db/20230119135807_etags.go

@@ -38,7 +38,7 @@ func init() {
 
 			for _, query := range queries {
 				if _, err := db.ExecContext(ctx, query); err != nil {
-					if !strings.Contains(err.Error(), "no such column2") {
+					if !strings.Contains(err.Error(), "no such column") {
 						return err
 					}
 				}

walrss/internal/db/20230119165107_lastmodified.go

@@ -0,0 +1,34 @@
+package db
+
+import (
+	"context"
+	"github.com/uptrace/bun"
+	"strings"
+)
+
+func init() {
+	migs.MustRegister(
+		func(ctx context.Context, db *bun.DB) error {
+			migLogger.Debug().Msg("2022-01-19@16:51:07 up")
+
+			if _, err := db.ExecContext(ctx, `ALTER TABLE feeds ADD COLUMN last_modified VARCHAR;`); err != nil {
+				if !strings.Contains(err.Error(), "duplicate column name") {
+					return err
+				}
+			}
+
+			return nil
+		},
+		func(ctx context.Context, db *bun.DB) error {
+			migLogger.Debug().Msg("2022-01-19@16:51:07 down")
+
+			if _, err := db.ExecContext(ctx, `ALTER TABLE feeds DROP COLUMN last_modified;`); err != nil {
+				if !strings.Contains(err.Error(), "no such column") {
+					return err
+				}
+			}
+
+			return nil
+		},
+	)
+}

walrss/internal/db/db.go

@@ -44,16 +44,24 @@ type Feed struct {
 	UserID string `bun:"user_id,notnull"`
 
 	LastEtag      string `bun:"last_etag,nullzero"`
+	LastModified  string `bun:"last_modified,nullzero"`
 	CachedContent string `bun:"cached_content,nullzero"`
 
 	User *User `bun:",rel:belongs-to,join:user_id=id"`
 }
 
 func (f *Feed) CacheWithEtag(etag, content string) {
+	f.LastModified = ""
 	f.LastEtag = etag
 	f.CachedContent = content
 }
 
+func (f *Feed) CacheWithLastModified(lastModified, content string) {
+	f.LastEtag = ""
+	f.LastModified = lastModified
+	f.CachedContent = content
+}
+
 type FeedSlice []*Feed
 
 func (f FeedSlice) Len() int {

walrss/internal/rss/processor.go

@@ -174,7 +174,7 @@ func getFeedContent(st *state.State, f *db.Feed) (*gofeed.Feed, error) {
 
 	requestBuilder := requests.URL(f.URL).ToBytesBuffer(buf).UserAgent(getUserAgent(st)).CopyHeaders(headers)
 
-	if f.LastEtag != "" {
+	if f.LastEtag != "" || f.LastModified != "" {
 		requestBuilder.AddValidator(
 			func(resp *http.Response) error {
 				if resp.StatusCode == http.StatusNotModified {
@@ -185,7 +185,13 @@ func getFeedContent(st *state.State, f *db.Feed) (*gofeed.Feed, error) {
 				}
 			},
 		)
-		requestBuilder.Header("If-None-Match", f.LastEtag)
+
+		if f.LastEtag != "" {
+			requestBuilder.Header("If-None-Match", f.LastEtag)
+		} else if f.LastModified != "" {
+			requestBuilder.Header("If-Modified-Since", f.LastModified)
+		}
+
 	} else {
 		requestBuilder.AddValidator(requests.DefaultValidator) // Since we're using CopyHeaders, we need to add the
 		// default validator back ourselves.
@@ -198,14 +204,17 @@ func getFeedContent(st *state.State, f *db.Feed) (*gofeed.Feed, error) {
 	if notModified {
 		log.Debug().Msgf("%s not modified", f.URL)
 		buf.WriteString(f.CachedContent)
-	} else {
-		log.Debug().Msgf("%s modified", f.URL)
-		etag := headers.Get("ETag")
-		if etag != "" {
-			f.CacheWithEtag(etag, buf.String())
-			if err := core.UpdateFeed(st, f); err != nil {
-				return nil, fmt.Errorf("failed to cache ETag-ed response: %v", err)
-			}
+	} else if etag := headers.Get("ETag"); etag != "" {
+		log.Debug().Msgf("%s modified (ETag)", f.URL)
+		f.CacheWithEtag(etag, buf.String())
+		if err := core.UpdateFeed(st, f); err != nil {
+			return nil, fmt.Errorf("failed to cache ETag-ed response: %v", err)
+		}
+	} else if lastModified := headers.Get("Last-Modified"); lastModified != "" {
+		log.Debug().Msgf("%s modified (Last-Modified)", f.URL)
+		f.CacheWithLastModified(lastModified, buf.String())
+		if err := core.UpdateFeed(st, f); err != nil {
+			return nil, fmt.Errorf("failed to cache Last-Modified enabled response: %v", err)
 		}
 	}