From 7086109d7879b64529578e3e16dcafbb8b57e842 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Thu, 21 Mar 2024 12:43:40 +1100 Subject: [PATCH] Change ffmpeg handling (#4688) * Make ffmpeg/ffprobe settable and remove auto download * Detect when ffmpeg not present in setup * Add download ffmpeg task * Add download ffmpeg button in system settings * Download ffmpeg during setup --- cmd/phasher/main.go | 10 +- graphql/schema/schema.graphql | 3 + graphql/schema/types/config.graphql | 8 + graphql/schema/types/metadata.graphql | 2 + internal/api/resolver_mutation_configure.go | 61 ++++- internal/api/resolver_query_configuration.go | 2 + internal/manager/config/config.go | 15 ++ internal/manager/init.go | 69 ++--- internal/manager/manager.go | 12 + internal/manager/models.go | 2 + internal/manager/task/download_ffmpeg.go | 241 ++++++++++++++++++ pkg/ffmpeg/codec_hardware.go | 2 +- pkg/ffmpeg/downloader.go | 228 +---------------- pkg/ffmpeg/ffmpeg.go | 89 +++++++ pkg/ffmpeg/ffprobe.go | 76 +++++- pkg/fsutil/file.go | 10 + ui/v2.5/graphql/data/config.graphql | 2 + ui/v2.5/graphql/mutations/config.graphql | 4 + .../graphql/queries/settings/metadata.graphql | 2 + .../Settings/SettingsSystemPanel.tsx | 48 +++- ui/v2.5/src/components/Setup/Setup.tsx | 88 ++++--- ui/v2.5/src/locales/en-GB.json | 17 +- 22 files changed, 694 insertions(+), 297 deletions(-) create mode 100644 internal/manager/task/download_ffmpeg.go diff --git a/cmd/phasher/main.go b/cmd/phasher/main.go index 059a1427d97..d4bf7959007 100644 --- a/cmd/phasher/main.go +++ b/cmd/phasher/main.go @@ -4,6 +4,7 @@ package main import ( "fmt" "os" + "os/exec" flag "github.com/spf13/pflag" "github.com/stashapp/stash/pkg/ffmpeg" @@ -45,6 +46,13 @@ func printPhash(ff *ffmpeg.FFMpeg, ffp ffmpeg.FFProbe, inputfile string, quiet * return nil } +func getPaths() (string, string) { + ffmpegPath, _ := exec.LookPath("ffmpeg") + ffprobePath, _ := exec.LookPath("ffprobe") + + return ffmpegPath, ffprobePath +} + func main() { flag.Usage = customUsage quiet := flag.BoolP("quiet", "q", false, "print only the phash") @@ -69,7 +77,7 @@ func main() { fmt.Fprintf(os.Stderr, "Example: parallel %v ::: *.mp4\n", os.Args[0]) } - ffmpegPath, ffprobePath := ffmpeg.GetPaths(nil) + ffmpegPath, ffprobePath := getPaths() encoder := ffmpeg.NewEncoder(ffmpegPath) // don't need to InitHWSupport, phashing doesn't use hw acceleration ffprobe := ffmpeg.FFProbe(ffprobePath) diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index f4866cb7166..2d7c57c0bfd 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -230,6 +230,9 @@ type Mutation { "Migrates the schema to the required version. Returns the job ID" migrate(input: MigrateInput!): ID! + "Downloads and installs ffmpeg and ffprobe binaries into the configuration directory. Returns the job ID." + downloadFFMpeg: ID! + sceneCreate(input: SceneCreateInput!): Scene sceneUpdate(input: SceneUpdateInput!): Scene sceneMerge(input: SceneMergeInput!): Scene diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 336385f290d..d98c9210065 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -81,6 +81,10 @@ input ConfigGeneralInput { blobsPath: String "Where to store blobs" blobsStorage: BlobsStorageType + "Path to the ffmpeg binary. If empty, stash will attempt to find it in the path or config directory" + ffmpegPath: String + "Path to the ffprobe binary. If empty, stash will attempt to find it in the path or config directory" + ffprobePath: String "Whether to calculate MD5 checksums for scene video files" calculateMD5: Boolean "Hash algorithm to use for generated file naming" @@ -199,6 +203,10 @@ type ConfigGeneralResult { blobsPath: String! "Where to store blobs" blobsStorage: BlobsStorageType! + "Path to the ffmpeg binary. If empty, stash will attempt to find it in the path or config directory" + ffmpegPath: String! + "Path to the ffprobe binary. If empty, stash will attempt to find it in the path or config directory" + ffprobePath: String! "Whether to calculate MD5 checksums for scene video files" calculateMD5: Boolean! "Hash algorithm to use for generated file naming" diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 73ded9c7a0a..983d0db5ffa 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -326,6 +326,8 @@ type SystemStatus { os: String! workingDir: String! homeDir: String! + ffmpegPath: String + ffprobePath: String } input MigrateInput { diff --git a/internal/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go index f5f3af47f40..c1cf987c90c 100644 --- a/internal/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -6,9 +6,12 @@ import ( "fmt" "path/filepath" "regexp" + "strconv" "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/internal/manager/task" + "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" @@ -22,6 +25,34 @@ func (r *mutationResolver) Setup(ctx context.Context, input manager.SetupInput) return err == nil, err } +func (r *mutationResolver) DownloadFFMpeg(ctx context.Context) (string, error) { + mgr := manager.GetInstance() + configDir := mgr.Config.GetConfigPath() + + // don't run if ffmpeg is already installed + ffmpegPath := ffmpeg.FindFFMpeg(configDir) + ffprobePath := ffmpeg.FindFFProbe(configDir) + if ffmpegPath != "" && ffprobePath != "" { + return "", fmt.Errorf("ffmpeg and ffprobe already installed at %s and %s", ffmpegPath, ffprobePath) + } + + t := &task.DownloadFFmpegJob{ + ConfigDirectory: configDir, + OnComplete: func(ctx context.Context) { + // clear the ffmpeg and ffprobe paths + logger.Infof("Clearing ffmpeg and ffprobe config paths so they are resolved from the config directory") + mgr.Config.Set(config.FFMpegPath, "") + mgr.Config.Set(config.FFProbePath, "") + mgr.RefreshFFMpeg(ctx) + mgr.RefreshStreamManager() + }, + } + + jobID := mgr.JobManager.Add(ctx, "Downloading ffmpeg...", t) + + return strconv.Itoa(jobID), nil +} + func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGeneralInput) (*ConfigGeneralResult, error) { c := config.GetInstance() @@ -161,12 +192,34 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen return makeConfigGeneralResult(), fmt.Errorf("blobs path must be set when using filesystem storage") } - // TODO - migrate between systems c.Set(config.BlobsStorage, input.BlobsStorage) refreshBlobStorage = true } + refreshFfmpeg := false + if input.FfmpegPath != nil && *input.FfmpegPath != c.GetFFMpegPath() { + if *input.FfmpegPath != "" { + if err := ffmpeg.ValidateFFMpeg(*input.FfmpegPath); err != nil { + return makeConfigGeneralResult(), fmt.Errorf("invalid ffmpeg path: %w", err) + } + } + + c.Set(config.FFMpegPath, input.FfmpegPath) + refreshFfmpeg = true + } + + if input.FfprobePath != nil && *input.FfprobePath != c.GetFFProbePath() { + if *input.FfprobePath != "" { + if err := ffmpeg.ValidateFFProbe(*input.FfprobePath); err != nil { + return makeConfigGeneralResult(), fmt.Errorf("invalid ffprobe path: %w", err) + } + } + + c.Set(config.FFProbePath, input.FfprobePath) + refreshFfmpeg = true + } + if input.VideoFileNamingAlgorithm != nil && *input.VideoFileNamingAlgorithm != c.GetVideoFileNamingAlgorithm() { calculateMD5 := c.IsCalculateMD5() if input.CalculateMd5 != nil { @@ -379,6 +432,12 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen if refreshPluginCache { manager.GetInstance().RefreshPluginCache() } + if refreshFfmpeg { + manager.GetInstance().RefreshFFMpeg(ctx) + + // refresh stream manager is required since ffmpeg changed + refreshStreamManager = true + } if refreshStreamManager { manager.GetInstance().RefreshStreamManager() } diff --git a/internal/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go index ce50f57f461..4e8b1b5b12d 100644 --- a/internal/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -91,6 +91,8 @@ func makeConfigGeneralResult() *ConfigGeneralResult { CachePath: config.GetCachePath(), BlobsPath: config.GetBlobsPath(), BlobsStorage: config.GetBlobsStorage(), + FfmpegPath: config.GetFFMpegPath(), + FfprobePath: config.GetFFProbePath(), CalculateMd5: config.IsCalculateMD5(), VideoFileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), ParallelTasks: config.GetParallelTasks(), diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index c5c5d7afdef..80c3b61cc2b 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -38,6 +38,9 @@ const ( Password = "password" MaxSessionAge = "max_session_age" + FFMpegPath = "ffmpeg_path" + FFProbePath = "ffprobe_path" + BlobsStorage = "blobs_storage" DefaultMaxSessionAge = 60 * 60 * 1 // 1 hours @@ -603,6 +606,18 @@ func (i *Config) GetBackupDirectoryPathOrDefault() string { return ret } +// GetFFMpegPath returns the path to the FFMpeg executable. +// If empty, stash will attempt to resolve it from the path. +func (i *Config) GetFFMpegPath() string { + return i.getString(FFMpegPath) +} + +// GetFFProbePath returns the path to the FFProbe executable. +// If empty, stash will attempt to resolve it from the path. +func (i *Config) GetFFProbePath() string { + return i.getString(FFProbePath) +} + func (i *Config) GetJWTSignKey() []byte { return []byte(i.getString(JWTSignKey)) } diff --git a/internal/manager/init.go b/internal/manager/init.go index a2e558b6a39..b184e8ce534 100644 --- a/internal/manager/init.go +++ b/internal/manager/init.go @@ -192,7 +192,6 @@ func (s *Manager) postInit(ctx context.Context) error { s.RefreshScraperCache() s.RefreshScraperSourceManager() - s.RefreshStreamManager() s.RefreshDLNA() s.SetBlobStoreOptions() @@ -239,9 +238,8 @@ func (s *Manager) postInit(ctx context.Context) error { logger.Info("Using HTTP proxy") } - if err := s.initFFmpeg(ctx); err != nil { - return fmt.Errorf("error initializing FFmpeg subsystem: %v", err) - } + s.RefreshFFMpeg(ctx) + s.RefreshStreamManager() return nil } @@ -260,41 +258,48 @@ func (s *Manager) writeStashIcon() { } } -func (s *Manager) initFFmpeg(ctx context.Context) error { +func (s *Manager) RefreshFFMpeg(ctx context.Context) { // use same directory as config path configDirectory := s.Config.GetConfigPath() - paths := []string{ - configDirectory, - paths.GetStashHomeDirectory(), - } - ffmpegPath, ffprobePath := ffmpeg.GetPaths(paths) - - if ffmpegPath == "" || ffprobePath == "" { - logger.Infof("couldn't find FFmpeg, attempting to download it") - if err := ffmpeg.Download(ctx, configDirectory); err != nil { - path, absErr := filepath.Abs(configDirectory) - if absErr != nil { - path = configDirectory - } - msg := `Unable to automatically download FFmpeg + stashHomeDir := paths.GetStashHomeDirectory() -Check the readme for download links. -The ffmpeg and ffprobe binaries should be placed in %s. + // prefer the configured paths + ffmpegPath := s.Config.GetFFMpegPath() + ffprobePath := s.Config.GetFFProbePath() -` - logger.Errorf(msg, path) - return err - } else { - // After download get new paths for ffmpeg and ffprobe - ffmpegPath, ffprobePath = ffmpeg.GetPaths(paths) + // ensure the paths are valid + if ffmpegPath != "" { + if err := ffmpeg.ValidateFFMpeg(ffmpegPath); err != nil { + logger.Errorf("invalid ffmpeg path: %v", err) + return + } + } else { + ffmpegPath = ffmpeg.ResolveFFMpeg(configDirectory, stashHomeDir) + } + + if ffprobePath != "" { + if err := ffmpeg.ValidateFFProbe(ffmpegPath); err != nil { + logger.Errorf("invalid ffprobe path: %v", err) + return } + } else { + ffprobePath = ffmpeg.ResolveFFProbe(configDirectory, stashHomeDir) } - s.FFMpeg = ffmpeg.NewEncoder(ffmpegPath) - s.FFProbe = ffmpeg.FFProbe(ffprobePath) + if ffmpegPath == "" { + logger.Warn("Couldn't find FFmpeg") + } + if ffprobePath == "" { + logger.Warn("Couldn't find FFProbe") + } - s.FFMpeg.InitHWSupport(ctx) - s.RefreshStreamManager() + if ffmpegPath != "" && ffprobePath != "" { + logger.Debugf("using ffmpeg: %s", ffmpegPath) + logger.Debugf("using ffprobe: %s", ffprobePath) - return nil + s.FFMpeg = ffmpeg.NewEncoder(ffmpegPath) + s.FFProbe = ffmpeg.FFProbe(ffprobePath) + + s.FFMpeg.InitHWSupport(ctx) + } } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 19b700344cb..a49831466a4 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -391,6 +391,16 @@ func (s *Manager) GetSystemStatus() *SystemStatus { configFile := s.Config.GetConfigFile() + ffmpegPath := "" + if s.FFMpeg != nil { + ffmpegPath = s.FFMpeg.Path() + } + + ffprobePath := "" + if s.FFProbe != "" { + ffprobePath = s.FFProbe.Path() + } + return &SystemStatus{ Os: runtime.GOOS, WorkingDir: workingDir, @@ -400,6 +410,8 @@ func (s *Manager) GetSystemStatus() *SystemStatus { AppSchema: appSchema, Status: status, ConfigPath: &configFile, + FfmpegPath: &ffmpegPath, + FfprobePath: &ffprobePath, } } diff --git a/internal/manager/models.go b/internal/manager/models.go index ab7a09bc07c..3e96e618287 100644 --- a/internal/manager/models.go +++ b/internal/manager/models.go @@ -13,6 +13,8 @@ type SystemStatus struct { Os string `json:"os"` WorkingDir string `json:"working_dir"` HomeDir string `json:"home_dir"` + FfmpegPath *string `json:"ffmpegPath"` + FfprobePath *string `json:"ffprobePath"` } type SetupInput struct { diff --git a/internal/manager/task/download_ffmpeg.go b/internal/manager/task/download_ffmpeg.go new file mode 100644 index 00000000000..30cb79c64bd --- /dev/null +++ b/internal/manager/task/download_ffmpeg.go @@ -0,0 +1,241 @@ +package task + +import ( + "archive/zip" + "context" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + + "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" +) + +type DownloadFFmpegJob struct { + ConfigDirectory string + OnComplete func(ctx context.Context) + urls []string + downloaded int +} + +func (s *DownloadFFmpegJob) Execute(ctx context.Context, progress *job.Progress) error { + if err := s.download(ctx, progress); err != nil { + if job.IsCancelled(ctx) { + return nil + } + return err + } + + if s.OnComplete != nil { + s.OnComplete(ctx) + } + + return nil +} + +func (s *DownloadFFmpegJob) setTaskProgress(taskProgress float64, progress *job.Progress) { + progress.SetPercent((float64(s.downloaded) + taskProgress) / float64(len(s.urls))) +} + +func (s *DownloadFFmpegJob) download(ctx context.Context, progress *job.Progress) error { + s.urls = ffmpeg.GetFFmpegURL() + + // set steps based on the number of URLs + + for _, url := range s.urls { + err := s.downloadSingle(ctx, url, progress) + if err != nil { + return err + } + s.downloaded++ + } + + // validate that the urls contained what we needed + executables := []string{fsutil.GetExeName("ffmpeg"), fsutil.GetExeName("ffprobe")} + for _, executable := range executables { + _, err := os.Stat(filepath.Join(s.ConfigDirectory, executable)) + if err != nil { + return err + } + } + return nil +} + +type downloadProgressReader struct { + io.Reader + setProgress func(taskProgress float64) + bytesRead int64 + total int64 +} + +func (r *downloadProgressReader) Read(p []byte) (int, error) { + read, err := r.Reader.Read(p) + if err == nil { + r.bytesRead += int64(read) + if r.total > 0 { + progress := float64(r.bytesRead) / float64(r.total) + r.setProgress(progress) + } + } + + return read, err +} + +func (s *DownloadFFmpegJob) downloadSingle(ctx context.Context, url string, progress *job.Progress) error { + if url == "" { + return fmt.Errorf("no ffmpeg url for this platform") + } + + configDirectory := s.ConfigDirectory + + // Configure where we want to download the archive + urlBase := path.Base(url) + archivePath := filepath.Join(configDirectory, urlBase) + _ = os.Remove(archivePath) // remove archive if it already exists + out, err := os.Create(archivePath) + if err != nil { + return err + } + defer out.Close() + + logger.Infof("Downloading %s...", url) + + progress.ExecuteTask(fmt.Sprintf("Downloading %s", url), func() { + err = s.downloadFile(ctx, url, out, progress) + }) + + if err != nil { + return fmt.Errorf("failed to download ffmpeg from %s: %w", url, err) + } + + logger.Info("Downloading complete") + + logger.Infof("Unzipping %s...", archivePath) + progress.ExecuteTask(fmt.Sprintf("Unzipping %s", archivePath), func() { + err = s.unzip(archivePath) + }) + + if err != nil { + return fmt.Errorf("failed to unzip ffmpeg archive: %w", err) + } + + // On OSX or Linux set downloaded files permissions + if runtime.GOOS == "darwin" || runtime.GOOS == "linux" { + _, err = os.Stat(filepath.Join(configDirectory, "ffmpeg")) + if !os.IsNotExist(err) { + if err = os.Chmod(filepath.Join(configDirectory, "ffmpeg"), 0755); err != nil { + return err + } + } + + _, err = os.Stat(filepath.Join(configDirectory, "ffprobe")) + if !os.IsNotExist(err) { + if err := os.Chmod(filepath.Join(configDirectory, "ffprobe"), 0755); err != nil { + return err + } + } + + // TODO: In future possible clear xattr to allow running on osx without user intervention + // TODO: this however may not be required. + // xattr -c /path/to/binary -- xattr.Remove(path, "com.apple.quarantine") + } + + return nil +} + +func (s *DownloadFFmpegJob) downloadFile(ctx context.Context, url string, out *os.File, progress *job.Progress) error { + // Make the HTTP request + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + + transport := &http.Transport{Proxy: http.ProxyFromEnvironment} + + client := &http.Client{ + Transport: transport, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check server response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + reader := &downloadProgressReader{ + Reader: resp.Body, + total: resp.ContentLength, + setProgress: func(taskProgress float64) { + s.setTaskProgress(taskProgress, progress) + }, + } + + // Write the response to the archive file location + if _, err := io.Copy(out, reader); err != nil { + return err + } + + mime := resp.Header.Get("Content-Type") + if mime != "application/zip" { // try detecting MIME type since some servers don't return the correct one + data := make([]byte, 500) // http.DetectContentType only reads up to 500 bytes + _, _ = out.ReadAt(data, 0) + mime = http.DetectContentType(data) + } + + if mime != "application/zip" { + return fmt.Errorf("downloaded file is not a zip archive") + } + + return nil +} + +func (s *DownloadFFmpegJob) unzip(src string) error { + zipReader, err := zip.OpenReader(src) + if err != nil { + return err + } + defer zipReader.Close() + + for _, f := range zipReader.File { + if f.FileInfo().IsDir() { + continue + } + filename := f.FileInfo().Name() + if filename != "ffprobe" && filename != "ffmpeg" && filename != "ffprobe.exe" && filename != "ffmpeg.exe" { + continue + } + + rc, err := f.Open() + if err != nil { + return err + } + + unzippedPath := filepath.Join(s.ConfigDirectory, filename) + unzippedOutput, err := os.Create(unzippedPath) + if err != nil { + return err + } + + _, err = io.Copy(unzippedOutput, rc) + if err != nil { + return err + } + + if err := unzippedOutput.Close(); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/ffmpeg/codec_hardware.go b/pkg/ffmpeg/codec_hardware.go index 6ab2e7870d2..e8f70033a8b 100644 --- a/pkg/ffmpeg/codec_hardware.go +++ b/pkg/ffmpeg/codec_hardware.go @@ -60,7 +60,7 @@ func (f *FFMpeg) InitHWSupport(ctx context.Context) { cmd.Stderr = &stderr if err := cmd.Start(); err != nil { - logger.Debugf("[InitHWSupport] error starting command: %w", err) + logger.Debugf("[InitHWSupport] error starting command: %v", err) continue } diff --git a/pkg/ffmpeg/downloader.go b/pkg/ffmpeg/downloader.go index d3af72bd710..407f38a3f0a 100644 --- a/pkg/ffmpeg/downloader.go +++ b/pkg/ffmpeg/downloader.go @@ -1,179 +1,10 @@ package ffmpeg import ( - "archive/zip" - "context" - "fmt" - "io" - "net/http" - "os" - "os/exec" - "path" - "path/filepath" "runtime" - "strings" - - stashExec "github.com/stashapp/stash/pkg/exec" - "github.com/stashapp/stash/pkg/fsutil" - "github.com/stashapp/stash/pkg/logger" ) -func GetPaths(paths []string) (string, string) { - var ffmpegPath, ffprobePath string - - // Check if ffmpeg exists in the PATH - if pathBinaryHasCorrectFlags() { - ffmpegPath, _ = exec.LookPath("ffmpeg") - ffprobePath, _ = exec.LookPath("ffprobe") - } - - // Check if ffmpeg exists in the config directory - if ffmpegPath == "" { - ffmpegPath = fsutil.FindInPaths(paths, getFFMpegFilename()) - } - if ffprobePath == "" { - ffprobePath = fsutil.FindInPaths(paths, getFFProbeFilename()) - } - - return ffmpegPath, ffprobePath -} - -func Download(ctx context.Context, configDirectory string) error { - for _, url := range getFFmpegURL() { - err := downloadSingle(ctx, configDirectory, url) - if err != nil { - return err - } - } - - // validate that the urls contained what we needed - executables := []string{getFFMpegFilename(), getFFProbeFilename()} - for _, executable := range executables { - _, err := os.Stat(filepath.Join(configDirectory, executable)) - if err != nil { - return err - } - } - return nil -} - -type progressReader struct { - io.Reader - lastProgress int64 - bytesRead int64 - total int64 -} - -func (r *progressReader) Read(p []byte) (int, error) { - read, err := r.Reader.Read(p) - if err == nil { - r.bytesRead += int64(read) - if r.total > 0 { - progress := int64(float64(r.bytesRead) / float64(r.total) * 100) - if progress/5 > r.lastProgress { - logger.Infof("%d%% downloaded...", progress) - r.lastProgress = progress / 5 - } - } - } - - return read, err -} - -func downloadSingle(ctx context.Context, configDirectory, url string) error { - if url == "" { - return fmt.Errorf("no ffmpeg url for this platform") - } - - // Configure where we want to download the archive - urlBase := path.Base(url) - archivePath := filepath.Join(configDirectory, urlBase) - _ = os.Remove(archivePath) // remove archive if it already exists - out, err := os.Create(archivePath) - if err != nil { - return err - } - defer out.Close() - - logger.Infof("Downloading %s...", url) - - // Make the HTTP request - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - - transport := &http.Transport{Proxy: http.ProxyFromEnvironment} - - client := &http.Client{ - Transport: transport, - } - - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - // Check server response - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status: %s", resp.Status) - } - - reader := &progressReader{ - Reader: resp.Body, - total: resp.ContentLength, - } - - // Write the response to the archive file location - _, err = io.Copy(out, reader) - if err != nil { - return err - } - logger.Info("Downloading complete") - - mime := resp.Header.Get("Content-Type") - if mime != "application/zip" { // try detecting MIME type since some servers don't return the correct one - data := make([]byte, 500) // http.DetectContentType only reads up to 500 bytes - _, _ = out.ReadAt(data, 0) - mime = http.DetectContentType(data) - } - - if mime == "application/zip" { - logger.Infof("Unzipping %s...", archivePath) - if err := unzip(archivePath, configDirectory); err != nil { - return err - } - - // On OSX or Linux set downloaded files permissions - if runtime.GOOS == "darwin" || runtime.GOOS == "linux" { - _, err = os.Stat(filepath.Join(configDirectory, "ffmpeg")) - if !os.IsNotExist(err) { - if err = os.Chmod(filepath.Join(configDirectory, "ffmpeg"), 0755); err != nil { - return err - } - } - - _, err = os.Stat(filepath.Join(configDirectory, "ffprobe")) - if !os.IsNotExist(err) { - if err := os.Chmod(filepath.Join(configDirectory, "ffprobe"), 0755); err != nil { - return err - } - } - - // TODO: In future possible clear xattr to allow running on osx without user intervention - // TODO: this however may not be required. - // xattr -c /path/to/binary -- xattr.Remove(path, "com.apple.quarantine") - } - - } else { - return fmt.Errorf("ffmpeg was downloaded to %s", archivePath) - } - - return nil -} - -func getFFmpegURL() []string { +func GetFFmpegURL() []string { var urls []string switch runtime.GOOS { case "darwin": @@ -208,60 +39,3 @@ func getFFProbeFilename() string { } return "ffprobe" } - -// Checks if ffmpeg in the path has the correct flags -func pathBinaryHasCorrectFlags() bool { - ffmpegPath, err := exec.LookPath("ffmpeg") - if err != nil { - return false - } - cmd := stashExec.Command(ffmpegPath) - bytes, _ := cmd.CombinedOutput() - output := string(bytes) - hasOpus := strings.Contains(output, "--enable-libopus") - hasVpx := strings.Contains(output, "--enable-libvpx") - hasX264 := strings.Contains(output, "--enable-libx264") - hasX265 := strings.Contains(output, "--enable-libx265") - hasWebp := strings.Contains(output, "--enable-libwebp") - return hasOpus && hasVpx && hasX264 && hasX265 && hasWebp -} - -func unzip(src, configDirectory string) error { - zipReader, err := zip.OpenReader(src) - if err != nil { - return err - } - defer zipReader.Close() - - for _, f := range zipReader.File { - if f.FileInfo().IsDir() { - continue - } - filename := f.FileInfo().Name() - if filename != "ffprobe" && filename != "ffmpeg" && filename != "ffprobe.exe" && filename != "ffmpeg.exe" { - continue - } - - rc, err := f.Open() - if err != nil { - return err - } - - unzippedPath := filepath.Join(configDirectory, filename) - unzippedOutput, err := os.Create(unzippedPath) - if err != nil { - return err - } - - _, err = io.Copy(unzippedOutput, rc) - if err != nil { - return err - } - - if err := unzippedOutput.Close(); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/ffmpeg/ffmpeg.go b/pkg/ffmpeg/ffmpeg.go index 58621bc7062..4303644a39d 100644 --- a/pkg/ffmpeg/ffmpeg.go +++ b/pkg/ffmpeg/ffmpeg.go @@ -3,11 +3,96 @@ package ffmpeg import ( "context" + "errors" + "fmt" "os/exec" + "strings" stashExec "github.com/stashapp/stash/pkg/exec" + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/logger" ) +func ValidateFFMpeg(ffmpegPath string) error { + cmd := stashExec.Command(ffmpegPath, "-h") + bytes, err := cmd.CombinedOutput() + output := string(bytes) + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("error running ffmpeg: %v", output) + } + + return fmt.Errorf("error running ffmpeg: %v", err) + } + + if !strings.Contains(output, "--enable-libopus") { + return fmt.Errorf("ffmpeg is missing libopus support") + } + if !strings.Contains(output, "--enable-libvpx") { + return fmt.Errorf("ffmpeg is missing libvpx support") + } + if !strings.Contains(output, "--enable-libx264") { + return fmt.Errorf("ffmpeg is missing libx264 support") + } + if !strings.Contains(output, "--enable-libx265") { + return fmt.Errorf("ffmpeg is missing libx265 support") + } + if !strings.Contains(output, "--enable-libwebp") { + return fmt.Errorf("ffmpeg is missing libwebp support") + } + return nil +} + +func LookPathFFMpeg() string { + ret, _ := exec.LookPath(getFFMpegFilename()) + + if ret != "" { + // ensure ffmpeg has the correct flags + if err := ValidateFFMpeg(ret); err != nil { + logger.Warnf("ffmpeg found in PATH (%s), but it is missing required flags: %v", ret, err) + ret = "" + } + } + + return ret +} + +func FindFFMpeg(path string) string { + ret := fsutil.FindInPaths([]string{path}, getFFMpegFilename()) + + if ret != "" { + // ensure ffmpeg has the correct flags + if err := ValidateFFMpeg(ret); err != nil { + logger.Warnf("ffmpeg found (%s), but it is missing required flags: %v", ret, err) + ret = "" + } + } + + return ret +} + +// ResolveFFMpeg attempts to resolve the path to the ffmpeg executable. +// It first looks in the provided path, then resolves from the environment, and finally looks in the fallback path. +// Returns an empty string if a valid ffmpeg cannot be found. +func ResolveFFMpeg(path string, fallbackPath string) string { + // look in the provided path first + ret := FindFFMpeg(path) + if ret != "" { + return ret + } + + // then resolve from the environment + ret = LookPathFFMpeg() + if ret != "" { + return ret + } + + // finally, look in the fallback path + ret = FindFFMpeg(fallbackPath) + return ret +} + // FFMpeg provides an interface to ffmpeg. type FFMpeg struct { ffmpeg string @@ -27,3 +112,7 @@ func NewEncoder(ffmpegPath string) *FFMpeg { func (f *FFMpeg) Command(ctx context.Context, args []string) *exec.Cmd { return stashExec.CommandContext(ctx, string(f.ffmpeg), args...) } + +func (f *FFMpeg) Path() string { + return f.ffmpeg +} diff --git a/pkg/ffmpeg/ffprobe.go b/pkg/ffmpeg/ffprobe.go index d7bda62dbcb..31b3cbf0029 100644 --- a/pkg/ffmpeg/ffprobe.go +++ b/pkg/ffmpeg/ffprobe.go @@ -2,17 +2,83 @@ package ffmpeg import ( "encoding/json" + "errors" "fmt" "math" "os" + "os/exec" "strconv" "strings" "time" - "github.com/stashapp/stash/pkg/exec" + stashExec "github.com/stashapp/stash/pkg/exec" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" ) +func ValidateFFProbe(ffprobePath string) error { + cmd := stashExec.Command(ffprobePath, "-h") + bytes, err := cmd.CombinedOutput() + output := string(bytes) + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("error running ffprobe: %v", output) + } + + return fmt.Errorf("error running ffprobe: %v", err) + } + + return nil +} + +func LookPathFFProbe() string { + ret, _ := exec.LookPath(getFFProbeFilename()) + + if ret != "" { + if err := ValidateFFProbe(ret); err != nil { + logger.Warnf("ffprobe found in PATH (%s), but it is missing required flags: %v", ret, err) + ret = "" + } + } + + return ret +} + +func FindFFProbe(path string) string { + ret := fsutil.FindInPaths([]string{path}, getFFProbeFilename()) + + if ret != "" { + if err := ValidateFFProbe(ret); err != nil { + logger.Warnf("ffprobe found (%s), but it is missing required flags: %v", ret, err) + ret = "" + } + } + + return ret +} + +// ResolveFFMpeg attempts to resolve the path to the ffmpeg executable. +// It first looks in the provided path, then resolves from the environment, and finally looks in the fallback path. +// Returns an empty string if a valid ffmpeg cannot be found. +func ResolveFFProbe(path string, fallbackPath string) string { + // look in the provided path first + ret := FindFFProbe(path) + if ret != "" { + return ret + } + + // then resolve from the environment + ret = LookPathFFProbe() + if ret != "" { + return ret + } + + // finally, look in the fallback path + ret = FindFFProbe(fallbackPath) + return ret +} + // VideoFile represents the ffprobe output for a video file. type VideoFile struct { JSON FFProbeJSON @@ -75,10 +141,14 @@ func (v *VideoFile) TranscodeScale(maxSize int) (int, int) { // FFProbe provides an interface to the ffprobe executable. type FFProbe string +func (f *FFProbe) Path() string { + return string(*f) +} + // NewVideoFile runs ffprobe on the given path and returns a VideoFile. func (f *FFProbe) NewVideoFile(videoPath string) (*VideoFile, error) { args := []string{"-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "-show_error", videoPath} - cmd := exec.Command(string(*f), args...) + cmd := stashExec.Command(string(*f), args...) out, err := cmd.Output() if err != nil { @@ -97,7 +167,7 @@ func (f *FFProbe) NewVideoFile(videoPath string) (*VideoFile, error) { // Used when the frame count is missing or incorrect. func (f *FFProbe) GetReadFrameCount(path string) (int64, error) { args := []string{"-v", "quiet", "-print_format", "json", "-count_frames", "-show_format", "-show_streams", "-show_error", path} - out, err := exec.Command(string(*f), args...).Output() + out, err := stashExec.Command(string(*f), args...).Output() if err != nil { return 0, fmt.Errorf("FFProbe encountered an error with <%s>.\nError JSON:\n%s\nError: %s", path, string(out), err.Error()) diff --git a/pkg/fsutil/file.go b/pkg/fsutil/file.go index 0c0eb52715f..19cf5ced65f 100644 --- a/pkg/fsutil/file.go +++ b/pkg/fsutil/file.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "github.com/stashapp/stash/pkg/logger" @@ -163,3 +164,12 @@ func SanitiseBasename(v string) string { return strings.TrimSpace(v) } + +// GetExeName returns the name of the given executable for the current platform. +// One windows it returns the name with the .exe extension. +func GetExeName(base string) string { + if runtime.GOOS == "windows" { + return base + ".exe" + } + return base +} diff --git a/ui/v2.5/graphql/data/config.graphql b/ui/v2.5/graphql/data/config.graphql index 160234d6e6a..3d4a00e6ece 100644 --- a/ui/v2.5/graphql/data/config.graphql +++ b/ui/v2.5/graphql/data/config.graphql @@ -13,6 +13,8 @@ fragment ConfigGeneralData on ConfigGeneralResult { cachePath blobsPath blobsStorage + ffmpegPath + ffprobePath calculateMD5 videoFileNamingAlgorithm parallelTasks diff --git a/ui/v2.5/graphql/mutations/config.graphql b/ui/v2.5/graphql/mutations/config.graphql index f6a8b47ce48..f786d0d8f56 100644 --- a/ui/v2.5/graphql/mutations/config.graphql +++ b/ui/v2.5/graphql/mutations/config.graphql @@ -6,6 +6,10 @@ mutation Migrate($input: MigrateInput!) { migrate(input: $input) } +mutation DownloadFFMpeg { + downloadFFMpeg +} + mutation ConfigureGeneral($input: ConfigGeneralInput!) { configureGeneral(input: $input) { ...ConfigGeneralData diff --git a/ui/v2.5/graphql/queries/settings/metadata.graphql b/ui/v2.5/graphql/queries/settings/metadata.graphql index 2081de96ec4..40cb1c53b19 100644 --- a/ui/v2.5/graphql/queries/settings/metadata.graphql +++ b/ui/v2.5/graphql/queries/settings/metadata.graphql @@ -8,5 +8,7 @@ query SystemStatus { os workingDir homeDir + ffmpegPath + ffprobePath } } diff --git a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx index a6d95d5a562..a3ab150dbc5 100644 --- a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx @@ -7,6 +7,7 @@ import { ModalSetting, NumberSetting, SelectSetting, + Setting, StringListSetting, StringSetting, } from "./Inputs"; @@ -15,12 +16,18 @@ import { VideoPreviewInput, VideoPreviewSettingsInput, } from "./GeneratePreviewOptions"; -import { useIntl } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; +import { Button } from "react-bootstrap"; +import { useToast } from "src/hooks/Toast"; +import { useHistory } from "react-router-dom"; export const SettingsConfigurationPanel: React.FC = () => { const intl = useIntl(); + const Toast = useToast(); + const history = useHistory(); const { general, loading, error, saveGeneral } = useSettings(); + const [mutateDownloadFFMpeg] = GQL.useDownloadFfMpegMutation(); const transcodeQualities = [ GQL.StreamingResolutionEnum.Low, @@ -107,6 +114,16 @@ export const SettingsConfigurationPanel: React.FC = () => { return "blobs_storage_type.database"; } + async function onDownloadFFMpeg() { + try { + await mutateDownloadFFMpeg(); + // navigate to tasks page to see the progress + history.push("/settings?tab=tasks"); + } catch (e) { + Toast.error(e); + } + } + if (error) return

{error.message}

; if (loading) return ; @@ -161,6 +178,35 @@ export const SettingsConfigurationPanel: React.FC = () => { onChange={(v) => saveGeneral({ customPerformerImageLocation: v })} /> + saveGeneral({ ffmpegPath: v })} + /> + + saveGeneral({ ffprobePath: v })} + /> + + + + + } + subHeadingID="config.general.ffmpeg.download_ffmpeg.description" + > + + + { const [blobsLocation, setBlobsLocation] = useState(""); const [loading, setLoading] = useState(false); const [setupError, setSetupError] = useState(); + const [downloadFFmpeg, setDownloadFFmpeg] = useState(true); const intl = useIntl(); const history = useHistory(); @@ -57,6 +58,8 @@ export const Setup: React.FC = () => { const { data: systemStatus, loading: statusLoading } = useSystemStatus(); const status = systemStatus?.systemStatus; + const [mutateDownloadFFMpeg] = GQL.useDownloadFfMpegMutation(); + const windows = status?.os === "windows"; const pathSep = windows ? "\\" : "/"; const homeDir = windows ? "%USERPROFILE%" : "$HOME"; @@ -164,7 +167,7 @@ export const Setup: React.FC = () => { ); } - function renderWelcomeSpecificConfig() { + const WelcomeSpecificConfig = () => { return ( <>
@@ -197,9 +200,9 @@ export const Setup: React.FC = () => {
); - } + }; - function renderWelcome() { + function DefaultWelcomeStep() { const homeDirPath = pathJoin(status?.homeDir ?? homeDir, ".stash"); return ( @@ -523,7 +526,7 @@ export const Setup: React.FC = () => { ); } - function renderSetPaths() { + function SetPathsStep() { return ( <> {maybeRenderStashAlert()} @@ -623,7 +626,7 @@ export const Setup: React.FC = () => { } } - function renderConfirm() { + function ConfirmStep() { let cfgDir: string; let config: string; if (overrideConfig) { @@ -735,7 +738,7 @@ export const Setup: React.FC = () => { ); } - function renderError() { + function ErrorStep() { function onBackClick() { setSetupError(undefined); goBack(2); @@ -771,7 +774,15 @@ export const Setup: React.FC = () => { ); } - function renderSuccess() { + function onFinishClick() { + if ((!status?.ffmpegPath || !status?.ffprobePath) && downloadFFmpeg) { + mutateDownloadFFMpeg(); + } + + history.push("/settings?tab=library"); + } + + function SuccessStep() { return ( <>
@@ -793,6 +804,28 @@ export const Setup: React.FC = () => { }} />

+ {!status?.ffmpegPath || !status?.ffprobePath ? ( + <> + + {chunks}, + }} + /> + +

+ setDownloadFFmpeg(!downloadFFmpeg)} + /> +

+ + ) : null}

@@ -838,23 +871,21 @@ export const Setup: React.FC = () => {

- - - +
); } - function renderFinish() { + function FinishStep() { if (setupError !== undefined) { - return renderError(); + return ; } - return renderSuccess(); + return ; } // only display setup wizard if system is not setup @@ -868,10 +899,11 @@ export const Setup: React.FC = () => { return ; } - const welcomeStep = overrideConfig - ? renderWelcomeSpecificConfig - : renderWelcome; - const steps = [welcomeStep, renderSetPaths, renderConfirm, renderFinish]; + const WelcomeStep = overrideConfig + ? WelcomeSpecificConfig + : DefaultWelcomeStep; + const steps = [WelcomeStep, SetPathsStep, ConfirmStep, FinishStep]; + const Step = steps[step]; function renderCreating() { return ( @@ -881,14 +913,6 @@ export const Setup: React.FC = () => { id: "setup.creating.creating_your_system", })} /> - - {chunks}, - }} - /> - ); } @@ -901,7 +925,13 @@ export const Setup: React.FC = () => {

- {loading ? renderCreating() : {steps[step]()}} + {loading ? ( + renderCreating() + ) : ( + + + + )} ); }; diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index eabbfe1ef7e..de9905813bc 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -316,6 +316,18 @@ "excluded_video_patterns_desc": "Regexps of video files/paths to exclude from Scan and add to Clean", "excluded_video_patterns_head": "Excluded Video Patterns", "ffmpeg": { + "download_ffmpeg": { + "description": "Downloads FFmpeg into the configuration directory and clears the ffmpeg and ffprobe paths to resolve from the configuration directory.", + "heading": "Download FFmpeg" + }, + "ffmpeg_path": { + "description": "Path to the ffmpeg executable (not just the folder). If empty, ffmpeg will be resolved from the environment via $PATH, the configuration directory, or from $HOME/.stash", + "heading": "FFmpeg Executable Path" + }, + "ffprobe_path": { + "description": "Path to the ffprobe executable (not just the folder). If empty, ffprobe will be resolved from the environment via $PATH, the configuration directory, or from $HOME/.stash", + "heading": "FFprobe Executable Path" + }, "hardware_acceleration": { "desc": "Uses available hardware to encode video for live transcoding.", "heading": "FFmpeg hardware encoding" @@ -1253,8 +1265,7 @@ "stash_library_directories": "Stash library directories" }, "creating": { - "creating_your_system": "Creating your system", - "ffmpeg_notice": "If ffmpeg is not yet in your paths, please be patient while stash downloads it. View the console output to see download progress." + "creating_your_system": "Creating your system" }, "errors": { "something_went_wrong": "Oh no! Something went wrong!", @@ -1303,9 +1314,11 @@ }, "stash_setup_wizard": "Stash Setup Wizard", "success": { + "download_ffmpeg": "Download ffmpeg", "getting_help": "Getting help", "help_links": "If you run into issues or have any questions or suggestions, feel free to open an issue in the {githubLink}, or ask the community in the {discordLink}.", "in_app_manual_explained": "You are encouraged to check out the in-app manual which can be accessed from the icon in the top-right corner of the screen that looks like this: {icon}", + "missing_ffmpeg": "You are missing the required ffmpeg binary. You can automatically download it into your configuration directory by checking the box below. Alternatively, you can supply paths to the ffmpeg and ffprobe binaries in the System Settings. These binaries must be present for Stash to function.", "next_config_step_one": "You will be taken to the Configuration page next. This page will allow you to customize what files to include and exclude, set a username and password to protect your system, and a whole bunch of other options.", "next_config_step_two": "When you are satisfied with these settings, you can begin scanning your content into Stash by clicking on {localized_task}, then {localized_scan}.", "open_collective": "Check out our {open_collective_link} to see how you can contribute to the continued development of Stash.",