Refactor xpath scraper code. Add fixed and map

go.sum

@@ -582,6 +582,7 @@ github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

pkg/api/resolver_mutation_scraper.go

@@ -3,11 +3,11 @@ package api
 import (
 	"context"
 
-	"github.com/stashapp/stash/pkg/scraper"
+	"github.com/stashapp/stash/pkg/manager"
 )
 
 func (r *mutationResolver) ReloadScrapers(ctx context.Context) (bool, error) {
-	err := scraper.ReloadScrapers()
+	err := manager.GetInstance().ScraperCache.ReloadScrapers()
 
 	if err != nil {
 		return false, err

pkg/api/resolver_query_scraper.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 
+	"github.com/stashapp/stash/pkg/manager"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scraper"
 )
@@ -12,12 +13,12 @@ func (r *queryResolver) ScrapeFreeones(ctx context.Context, performer_name strin
 	scrapedPerformer := models.ScrapedPerformerInput{
 		Name: &performer_name,
 	}
-	return scraper.GetFreeonesScraper().ScrapePerformer(scrapedPerformer)
+	return manager.GetInstance().ScraperCache.ScrapePerformer(scraper.FreeonesScraperID, scrapedPerformer)
 }
 
 // deprecated
 func (r *queryResolver) ScrapeFreeonesPerformerList(ctx context.Context, query string) ([]string, error) {
-	scrapedPerformers, err := scraper.GetFreeonesScraper().ScrapePerformerNames(query)
+	scrapedPerformers, err := manager.GetInstance().ScraperCache.ScrapePerformerList(scraper.FreeonesScraperID, query)
 
 	if err != nil {
 		return nil, err
@@ -33,33 +34,33 @@ func (r *queryResolver) ScrapeFreeonesPerformerList(ctx context.Context, query s
 }
 
 func (r *queryResolver) ListPerformerScrapers(ctx context.Context) ([]*models.Scraper, error) {
-	return scraper.ListPerformerScrapers()
+	return manager.GetInstance().ScraperCache.ListPerformerScrapers(), nil
 }
 
 func (r *queryResolver) ListSceneScrapers(ctx context.Context) ([]*models.Scraper, error) {
-	return scraper.ListSceneScrapers()
+	return manager.GetInstance().ScraperCache.ListSceneScrapers(), nil
 }
 
 func (r *queryResolver) ScrapePerformerList(ctx context.Context, scraperID string, query string) ([]*models.ScrapedPerformer, error) {
 	if query == "" {
 		return nil, nil
 	}
 
-	return scraper.ScrapePerformerList(scraperID, query)
+	return manager.GetInstance().ScraperCache.ScrapePerformerList(scraperID, query)
 }
 
 func (r *queryResolver) ScrapePerformer(ctx context.Context, scraperID string, scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) {
-	return scraper.ScrapePerformer(scraperID, scrapedPerformer)
+	return manager.GetInstance().ScraperCache.ScrapePerformer(scraperID, scrapedPerformer)
 }
 
 func (r *queryResolver) ScrapePerformerURL(ctx context.Context, url string) (*models.ScrapedPerformer, error) {
-	return scraper.ScrapePerformerURL(url)
+	return manager.GetInstance().ScraperCache.ScrapePerformerURL(url)
 }
 
 func (r *queryResolver) ScrapeScene(ctx context.Context, scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error) {
-	return scraper.ScrapeScene(scraperID, scene)
+	return manager.GetInstance().ScraperCache.ScrapeScene(scraperID, scene)
 }
 
 func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) {
-	return scraper.ScrapeSceneURL(url)
+	return manager.GetInstance().ScraperCache.ScrapeSceneURL(url)
 }

pkg/manager/manager.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/manager/paths"
+	"github.com/stashapp/stash/pkg/scraper"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
@@ -20,6 +21,8 @@ type singleton struct {
 
 	FFMPEGPath  string
 	FFProbePath string
+
+	ScraperCache *scraper.Cache
 }
 
 var instance *singleton
@@ -47,6 +50,8 @@ func Initialize() *singleton {
 			Status: TaskStatus{Status: Idle, Progress: -1},
 			Paths:  paths.NewPaths(),
 			JSON:   &jsonUtils{},
+
+			ScraperCache: initScraperCache(),
 		}
 
 		instance.RefreshConfig()
@@ -146,6 +151,20 @@ func initLog() {
 	logger.Init(config.GetLogFile(), config.GetLogOut(), config.GetLogLevel())
 }
 
+func initScraperCache() *scraper.Cache {
+	scraperConfig := scraper.GlobalConfig{
+		Path:      config.GetScrapersPath(),
+		UserAgent: config.GetScraperUserAgent(),
+	}
+	ret, err := scraper.NewCache(scraperConfig)
+
+	if err != nil {
+		logger.Errorf("Error reading scraper configs: %s", err.Error())
+	}
+
+	return ret
+}
+
 func (s *singleton) RefreshConfig() {
 	s.Paths = paths.NewPaths()
 	if config.IsValid() {

pkg/scraper/action.go

@@ -0,0 +1,53 @@
+package scraper
+
+import "github.com/stashapp/stash/pkg/models"
+
+type scraperAction string
+
+const (
+	scraperActionScript scraperAction = "script"
+	scraperActionStash  scraperAction = "stash"
+	scraperActionXPath  scraperAction = "scrapeXPath"
+)
+
+var allScraperAction = []scraperAction{
+	scraperActionScript,
+	scraperActionStash,
+	scraperActionXPath,
+}
+
+func (e scraperAction) IsValid() bool {
+	switch e {
+	case scraperActionScript, scraperActionStash, scraperActionXPath:
+		return true
+	}
+	return false
+}
+
+type scrapeOptions struct {
+	scraper      scraperTypeConfig
+	config       config
+	globalConfig GlobalConfig
+}
+
+type scraper interface {
+	scrapePerformersByName(name string) ([]*models.ScrapedPerformer, error)
+	scrapePerformerByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error)
+	scrapePerformerByURL(url string) (*models.ScrapedPerformer, error)
+
+	scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error)
+	scrapeSceneByURL(url string) (*models.ScrapedScene, error)
+}
+
+func getScraper(scraper scraperTypeConfig, config config, globalConfig GlobalConfig) scraper {
+	switch scraper.Action {
+	case scraperActionScript:
+		return newScriptScraper(scraper, config, globalConfig)
+	case scraperActionStash:
+		return newStashScraper(scraper, config, globalConfig)
+	case scraperActionXPath:
+		return newXpathScraper(scraper, config, globalConfig)
+	}
+
+	panic("unknown scraper action: " + scraper.Action)
+}