Merge pull request #43 from satisfactorymodding/staging

feat: mod asset listing

Author: Vilsol

db/postgres/mod.go

@@ -121,7 +121,7 @@ func GetModCountNew(ctx context.Context, filter *models.ModFilter, unapproved bo
 
 func IncrementModViews(ctx context.Context, mod *Mod) {
 	// TODO unignore
-	//DBCtx(ctx).Model(mod).Update("views", mod.Views+1)
+	// DBCtx(ctx).Model(mod).Update("views", mod.Views+1)
 }
 
 func GetMods(ctx context.Context, limit int, offset int, orderBy string, order string, search string, unapproved bool) []Mod {

gql/gql_utils.go

@@ -2,11 +2,14 @@ package gql
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/satisfactorymodding/smr-api/db/postgres"
@@ -27,22 +30,22 @@ func WrapMutationTrace(ctx context.Context, action string) (TraceWrapper, contex
 }
 
 func wrapTrace(ctx context.Context, action string, actionType string) (TraceWrapper, context.Context) {
-	// spanCtx, span := otel.Tracer("gql").Start(ctx, "GraphQL "+action, trace.WithAttributes(
-	//	attribute.String("action_type", "API.graphql."+actionType),
-	//))
+	spanCtx, span := otel.Tracer("gql").Start(ctx, "GraphQL "+action, trace.WithAttributes(
+		attribute.String("action_type", "API.graphql."+actionType),
+	))
 
 	return TraceWrapper{
-		// Span: span,
-	}, ctx
+		Span: span,
+	}, spanCtx
 }
 
 func (wrapper TraceWrapper) end() {
-	// defer wrapper.Span.End()
-	//
-	//if err := recover(); err != nil {
-	//	wrapper.Span.RecordError(fmt.Errorf("panic: %v", err))
-	//	panic(err)
-	//}
+	defer wrapper.Span.End()
+
+	if err := recover(); err != nil {
+		wrapper.Span.RecordError(fmt.Errorf("panic: %v", err))
+		panic(err)
+	}
 }
 
 // SetStringINNOE sets target if value not nil or empty

gql/resolver_mods.go

@@ -619,3 +619,22 @@ func (r *queryResolver) ResolveModVersions(ctx context.Context, filter []*genera
 
 	return modVersions, nil
 }
+
+func (r *queryResolver) GetModAssetList(ctx context.Context, modReference string) ([]string, error) {
+	wrapper, _ := WrapQueryTrace(ctx, "getModAssetList")
+	defer wrapper.end()
+
+	list := redis.GetModAssetList(modReference)
+	if list != nil {
+		return list, nil
+	}
+
+	assets, err := storage.ListModAssets(modReference)
+	if err != nil {
+		return nil, err
+	}
+
+	redis.StoreModAssetList(modReference, assets)
+
+	return assets, nil
+}

migrations/code/20240108075200_new_parser.go

@@ -0,0 +1,25 @@
+package code
+
+import (
+	"context"
+	"strings"
+
+	"github.com/lab259/go-migration"
+	"github.com/rs/zerolog/log"
+
+	"github.com/satisfactorymodding/smr-api/db/postgres"
+	"github.com/satisfactorymodding/smr-api/migrations/utils"
+)
+
+func init() {
+	migration.NewCodeMigration(
+		func(executionContext interface{}) error {
+			ctx := log.Logger.WithContext(context.TODO())
+			utils.ReindexAllModFiles(ctx, true, nil, func(version postgres.Version) bool {
+				smlVersion := version.SMLVersion
+				return strings.Contains(smlVersion, "3.6.1") || strings.Contains(smlVersion, "3.7.0")
+			})
+			return nil
+		},
+	)
+}

redis/redis.go

@@ -1,11 +1,14 @@
 package redis
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"io"
 	"strconv"
 	"time"
 
@@ -140,3 +143,36 @@ func GetVersionUploadState(versionID string) (*generated.CreateVersionResponse,
 func FlushRedis() {
 	client.FlushDB()
 }
+
+func StoreModAssetList(modReference string, assets []string) {
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+
+	b, _ := json.Marshal(assets)
+	_, _ = zw.Write(b)
+	_ = zw.Close()
+
+	client.Set(fmt.Sprintf("assets:%s", modReference), buf.Bytes(), time.Hour*24)
+}
+
+func GetModAssetList(modReference string) []string {
+	result, err := client.Get(fmt.Sprintf("assets:%s", modReference)).Result()
+	if err != nil {
+		return nil
+	}
+
+	reader, err := gzip.NewReader(bytes.NewReader([]byte(result)))
+	if err != nil {
+		return nil
+	}
+
+	all, err := io.ReadAll(reader)
+	if err != nil {
+		return nil
+	}
+
+	out := make([]string, 0)
+	_ = json.Unmarshal(all, &out)
+
+	return out
+}

schemas/mod.graphql

@@ -119,6 +119,8 @@ extend type Query {
     getMyUnapprovedMods(filter: ModFilter): GetMyMods! @isLoggedIn
 
     resolveModVersions(filter: [ModVersionConstraint!]!): [ModVersion!]!
+
+    getModAssetList(modReference: ModID!): [String!]!
 }
 
 ### Mutations

storage/b2.go

@@ -223,6 +223,24 @@ func (b2o *B2) Meta(key string) (*ObjectMeta, error) {
 	}, nil
 }
 
-func (b2o *B2) List(key string) ([]Object, error) {
-	return nil, nil // no-op
+func (b2o *B2) List(prefix string) ([]Object, error) {
+	out := make([]Object, 0)
+
+	err := b2o.S3Client.ListObjectsPages(&s3.ListObjectsInput{
+		Bucket: aws.String(b2o.Config.Bucket),
+		Prefix: aws.String(prefix),
+	}, func(output *s3.ListObjectsOutput, b bool) bool {
+		for _, obj := range output.Contents {
+			out = append(out, Object{
+				Key:          obj.Key,
+				LastModified: obj.LastModified,
+			})
+		}
+		return true
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list objects")
+	}
+
+	return out, nil
 }

storage/s3.go

@@ -248,21 +248,23 @@ func (s3o *S3) Meta(key string) (*ObjectMeta, error) {
 }
 
 func (s3o *S3) List(prefix string) ([]Object, error) {
-	objects, err := s3o.S3Client.ListObjects(&s3.ListObjectsInput{
+	out := make([]Object, 0)
+
+	err := s3o.S3Client.ListObjectsPages(&s3.ListObjectsInput{
 		Bucket: aws.String(viper.GetString("storage.bucket")),
 		Prefix: aws.String(prefix),
+	}, func(output *s3.ListObjectsOutput, b bool) bool {
+		for _, obj := range output.Contents {
+			out = append(out, Object{
+				Key:          obj.Key,
+				LastModified: obj.LastModified,
+			})
+		}
+		return true
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to list objects")
 	}
 
-	out := make([]Object, len(objects.Contents))
-	for i, obj := range objects.Contents {
-		out[i] = Object{
-			Key:          obj.Key,
-			LastModified: obj.LastModified,
-		}
-	}
-
 	return out, nil
 }

storage/storage.go

@@ -8,6 +8,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"sort"
 	"strings"
 	"time"
 
@@ -445,7 +446,7 @@ func copyModFileToArchZip(file *zip.File, zipWriter *zip.Writer, newName string)
 }
 
 func DeleteOldModAssets(modReference string, before time.Time) {
-	list, err := storage.List(fmt.Sprintf("/assets/mods/%s", modReference))
+	list, err := storage.List(fmt.Sprintf("assets/mods/%s", modReference))
 	if err != nil {
 		log.Err(err).Msg("failed to list assets")
 		return
@@ -477,3 +478,27 @@ func UploadModAsset(ctx context.Context, modReference string, path string, data
 		log.Err(err).Str("path", path).Msg("failed to upload mod asset")
 	}
 }
+
+func ListModAssets(modReference string) ([]string, error) {
+	if storage == nil {
+		return nil, errors.New("no storage defined")
+	}
+
+	list, err := storage.List(fmt.Sprintf("assets/mods/%s", modReference))
+	if err != nil {
+		return nil, errors.New("failed to list assets")
+	}
+
+	out := make([]string, len(list))
+	for i, object := range list {
+		if object.Key == nil {
+			continue
+		}
+
+		out[i] = *object.Key
+	}
+
+	sort.Strings(out)
+
+	return out, nil
+}