library entry save data

beclab · May 6, 2024 · c53f301 · c53f301
1 parent 0693ef8
commit c53f301
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 4 deletions.
diff --git a/backend-server/api/entry.go b/backend-server/api/entry.go
@@ -60,7 +60,7 @@ func (h *handler) newFetchContent(entry *model.Entry) string {
 			feedSearchRSSList = append(feedSearchRSSList, feedNotification)
 		}
 	}
-	entry = crawler.EntryCrawler(entry, feedUrl, userAgent, cookie, certificates, fetchViaProxy)
+	crawler.EntryCrawler(entry, feedUrl, userAgent, cookie, certificates, fetchViaProxy)
 
 	notificationData := model.NotificationData{
 		Name:      entry.Title,

diff --git a/backend-server/crawler/entry.go b/backend-server/crawler/entry.go
@@ -12,7 +12,7 @@ import (
 	"go.uber.org/zap"
 )
 
-func EntryCrawler(entry *model.Entry, feedUrl, userAgent, cookie string, certificates, fetchViaProxy bool) *model.Entry {
+func EntryCrawler(entry *model.Entry, feedUrl, userAgent, cookie string, certificates, fetchViaProxy bool) {
 	//entryID, entryUrl, entryTitle, imageUrl, author string, entryPublishedAt int64, feed *model.Feed) (string, string, int64) {
 
 	entry.RawContent = fetchRawContnt(
@@ -65,7 +65,6 @@ func EntryCrawler(entry *model.Entry, feedUrl, userAgent, cookie string, certifi
 	} else {
 		common.Logger.Error("crawler raw content is null", zap.String("url", entry.URL))
 	}
-	return entry
 	//return rawContent, rtContent, entryPublishedAt
 }
 

diff --git a/backend-server/storage/entry.go b/backend-server/storage/entry.go
@@ -96,7 +96,7 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
 func (s *Storage) UpdateEntryDocID(entry *model.Entry) error {
 	coll := getEntryMongodbColl(s)
 	filter := bson.M{"_id": entry.ID}
-	update := bson.M{"$set": bson.M{"doc_id": entry.DocId}}
+	update := bson.M{"$set": bson.M{"crawler": true, "language": entry.Language, "author": entry.Author, "title": entry.Title, "raw_content": entry.RawContent, "full_content": entry.FullContent, "doc_id": entry.DocId}}
 
 	if _, err := coll.UpdateOne(context.TODO(), filter, update); err != nil {
 		common.Logger.Error("unable to update entry", zap.String("url", entry.ID.Hex()), zap.Error(err))