Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions internal/aliases/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,15 @@ func (p *Provider) NativeFileProviderTypes() []string {
return nil
}

// NativeBatchProviderTypes delegates provider capability inventory to the inner
// provider when available.
func (p *Provider) NativeBatchProviderTypes() []string {
if typed, ok := p.inner.(core.NativeBatchProviderTypeLister); ok {
return typed.NativeBatchProviderTypes()
}
return nil
}

// NativeResponseProviderTypes delegates provider capability inventory to the
// inner provider when available.
func (p *Provider) NativeResponseProviderTypes() []string {
Expand Down
80 changes: 59 additions & 21 deletions internal/app/app.go

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions internal/core/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ type NativeBatchRoutableProvider interface {
GetBatchResults(ctx context.Context, providerType, id string) (*BatchResultsResponse, error)
}

// NativeBatchProviderTypeLister exposes registered provider types that support
// native batch operations.
type NativeBatchProviderTypeLister interface {
NativeBatchProviderTypes() []string
}

// NativeBatchHintRoutableProvider is an optional routing extension for
// providers that can consume persisted per-item endpoint hints.
type NativeBatchHintRoutableProvider interface {
Expand Down
80 changes: 80 additions & 0 deletions internal/filestore/factory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package filestore

import (
"context"
"database/sql"
"errors"
"fmt"

"github.com/jackc/pgx/v5/pgxpool"
"go.mongodb.org/mongo-driver/v2/mongo"

"gomodel/config"
"gomodel/internal/storage"
)

// Result holds the initialized file store and optional owned storage.
type Result struct {
Store Store
Storage storage.Storage
}

// Close releases resources held by the file store.
func (r *Result) Close() error {
if r == nil {
return nil
}
var errs []error
if r.Store != nil {
if err := r.Store.Close(); err != nil {
errs = append(errs, fmt.Errorf("store close: %w", err))
}
}
if r.Storage != nil {
if err := r.Storage.Close(); err != nil {
errs = append(errs, fmt.Errorf("storage close: %w", err))
}
}
if len(errs) > 0 {
return fmt.Errorf("close errors: %w", errors.Join(errs...))
}
return nil
}

// New creates a file store from app configuration.
func New(ctx context.Context, cfg *config.Config) (*Result, error) {
if cfg == nil {
return nil, fmt.Errorf("config is required")
}
store, err := storage.New(ctx, cfg.Storage.BackendConfig())
if err != nil {
return nil, fmt.Errorf("failed to create storage: %w", err)
}
fileStore, err := createStore(ctx, store)
if err != nil {
_ = store.Close()
return nil, err
}
return &Result{Store: fileStore, Storage: store}, nil
}

// NewWithSharedStorage creates a file store using a shared storage connection.
func NewWithSharedStorage(ctx context.Context, shared storage.Storage) (*Result, error) {
if shared == nil {
return nil, fmt.Errorf("shared storage is required")
}
fileStore, err := createStore(ctx, shared)
if err != nil {
return nil, err
}
return &Result{Store: fileStore}, nil
}

func createStore(ctx context.Context, store storage.Storage) (Store, error) {
return storage.ResolveBackend[Store](
store,
func(db *sql.DB) (Store, error) { return NewSQLiteStore(db) },
func(pool *pgxpool.Pool) (Store, error) { return NewPostgreSQLStore(ctx, pool) },
func(db *mongo.Database) (Store, error) { return NewMongoDBStore(db) },
)
}
75 changes: 75 additions & 0 deletions internal/filestore/store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Package filestore persists provider ownership for uploaded files.
package filestore

import (
"context"
"errors"
"fmt"
"strings"
)

// ErrNotFound indicates a requested file mapping was not found.
var ErrNotFound = errors.New("file mapping not found")

// StoredFile maps an OpenAI-compatible file ID to the provider that owns it.
type StoredFile struct {
ID string `json:"id" bson:"_id"`
ProviderType string `json:"provider_type" bson:"provider_type"`
Purpose string `json:"purpose,omitempty" bson:"purpose,omitempty"`
Filename string `json:"filename,omitempty" bson:"filename,omitempty"`
Bytes int64 `json:"bytes,omitempty" bson:"bytes,omitempty"`
CreatedAt int64 `json:"created_at,omitempty" bson:"created_at,omitempty"`
UserPath string `json:"user_path,omitempty" bson:"user_path,omitempty"`
}

// Store defines persistence operations for file provider mappings.
type Store interface {
Upsert(ctx context.Context, file *StoredFile) error
Get(ctx context.Context, id string) (*StoredFile, error)
Delete(ctx context.Context, id string) error
Close() error
}

type rowScanner interface {
Scan(dest ...any) error
}

func scanStoredFile(row rowScanner, notFound error) (*StoredFile, error) {
file := &StoredFile{}
err := row.Scan(&file.ID, &file.ProviderType, &file.Purpose, &file.Filename, &file.Bytes, &file.CreatedAt, &file.UserPath)
if err != nil {
if errors.Is(err, notFound) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("query file mapping: %w", err)
}
return cloneStoredFile(file)
}

func normalizeStoredFile(file *StoredFile) (*StoredFile, error) {
if file == nil {
return nil, fmt.Errorf("file mapping is nil")
}
normalized := *file
normalized.ID = strings.TrimSpace(normalized.ID)
normalized.ProviderType = strings.TrimSpace(normalized.ProviderType)
normalized.Purpose = strings.TrimSpace(normalized.Purpose)
normalized.Filename = strings.TrimSpace(normalized.Filename)
normalized.UserPath = strings.TrimSpace(normalized.UserPath)
if normalized.ID == "" {
return nil, fmt.Errorf("file id is required")
}
if normalized.ProviderType == "" {
return nil, fmt.Errorf("provider type is required")
}
return &normalized, nil
}

func cloneStoredFile(file *StoredFile) (*StoredFile, error) {
normalized, err := normalizeStoredFile(file)
if err != nil {
return nil, err
}
cloned := *normalized
return &cloned, nil
}
59 changes: 59 additions & 0 deletions internal/filestore/store_memory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package filestore

import (
"context"
"sync"
)

// MemoryStore keeps file provider mappings in process memory.
type MemoryStore struct {
mu sync.RWMutex
items map[string]*StoredFile
}

// NewMemoryStore creates an empty in-memory file store.
func NewMemoryStore() *MemoryStore {
return &MemoryStore{items: make(map[string]*StoredFile)}
}

// Upsert creates or replaces a file mapping.
func (s *MemoryStore) Upsert(_ context.Context, file *StoredFile) error {
cloned, err := cloneStoredFile(file)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if existing, ok := s.items[cloned.ID]; ok {
cloned.CreatedAt = existing.CreatedAt
}
s.items[cloned.ID] = cloned
return nil
}

// Get retrieves one file mapping by id.
func (s *MemoryStore) Get(_ context.Context, id string) (*StoredFile, error) {
s.mu.RLock()
file, ok := s.items[id]
s.mu.RUnlock()
if !ok {
return nil, ErrNotFound
}
return cloneStoredFile(file)
}

// Delete removes one file mapping by id.
func (s *MemoryStore) Delete(_ context.Context, id string) error {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.items[id]; !ok {
return ErrNotFound
}
delete(s.items, id)
return nil
}

// Close releases resources.
func (s *MemoryStore) Close() error {
return nil
}
108 changes: 108 additions & 0 deletions internal/filestore/store_mongodb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package filestore

import (
"context"
"errors"
"fmt"
"time"

"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
)

type mongoFileDocument struct {
ID string `bson:"_id"`
ProviderType string `bson:"provider_type"`
Purpose string `bson:"purpose,omitempty"`
Filename string `bson:"filename,omitempty"`
Bytes int64 `bson:"bytes,omitempty"`
CreatedAt int64 `bson:"created_at,omitempty"`
UpdatedAt int64 `bson:"updated_at"`
UserPath string `bson:"user_path,omitempty"`
}

// MongoDBStore stores file provider mappings in MongoDB.
type MongoDBStore struct {
collection *mongo.Collection
}

// NewMongoDBStore creates collection indexes if needed.
func NewMongoDBStore(database *mongo.Database) (*MongoDBStore, error) {
if database == nil {
return nil, fmt.Errorf("database is required")
}
coll := database.Collection("file_mappings")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
indexes := []mongo.IndexModel{
{Keys: bson.D{{Key: "provider_type", Value: 1}}},
{Keys: bson.D{{Key: "created_at", Value: -1}}},
}
if _, err := coll.Indexes().CreateMany(ctx, indexes); err != nil {
return nil, fmt.Errorf("create file_mappings indexes: %w", err)
}
return &MongoDBStore{collection: coll}, nil
}

// Upsert creates or replaces a file mapping.
func (s *MongoDBStore) Upsert(ctx context.Context, file *StoredFile) error {
normalized, err := normalizeStoredFile(file)
if err != nil {
return err
}
opts := options.UpdateOne().SetUpsert(true)
update := bson.M{
"$set": bson.M{
"provider_type": normalized.ProviderType,
"purpose": normalized.Purpose,
"filename": normalized.Filename,
"bytes": normalized.Bytes,
"updated_at": time.Now().Unix(),
"user_path": normalized.UserPath,
},
"$setOnInsert": bson.M{"_id": normalized.ID, "created_at": normalized.CreatedAt},
}
if _, err := s.collection.UpdateOne(ctx, bson.M{"_id": normalized.ID}, update, opts); err != nil {
return fmt.Errorf("upsert file mapping: %w", err)
}
return nil
}

// Get retrieves one file mapping by id.
func (s *MongoDBStore) Get(ctx context.Context, id string) (*StoredFile, error) {
var doc mongoFileDocument
err := s.collection.FindOne(ctx, bson.M{"_id": id}).Decode(&doc)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("query file mapping: %w", err)
}
return cloneStoredFile(&StoredFile{
ID: doc.ID,
ProviderType: doc.ProviderType,
Purpose: doc.Purpose,
Filename: doc.Filename,
Bytes: doc.Bytes,
CreatedAt: doc.CreatedAt,
UserPath: doc.UserPath,
})
}

// Delete removes one file mapping by id.
func (s *MongoDBStore) Delete(ctx context.Context, id string) error {
result, err := s.collection.DeleteOne(ctx, bson.M{"_id": id})
if err != nil {
return fmt.Errorf("delete file mapping: %w", err)
}
if result.DeletedCount == 0 {
return ErrNotFound
}
return nil
}

// Close is a no-op; Mongo client lifecycle is managed by storage layer.
func (s *MongoDBStore) Close() error {
return nil
}
Loading
Loading