Merge branch 'master' into StableCascade

TensorStack-AI · Apr 25, 2024 · b6920e0 · b6920e0
2 parents 6a62767 + dc2ed2f
commit b6920e0
Show file tree

Hide file tree

Showing 26 changed files with 139 additions and 1 deletion.
diff --git a/OnnxStack.Console/Examples/ModelDebug.cs b/OnnxStack.Console/Examples/ModelDebug.cs
@@ -0,0 +1,63 @@
+using OnnxStack.Core.Image;
+using OnnxStack.StableDiffusion.Config;
+using OnnxStack.StableDiffusion.Enums;
+using OnnxStack.StableDiffusion.Pipelines;
+
+namespace OnnxStack.Console.Runner
+{
+    public sealed class ModelDebug : IExampleRunner
+    {
+        private readonly string _outputDirectory;
+        private readonly StableDiffusionConfig _configuration;
+
+        public ModelDebug(StableDiffusionConfig configuration)
+        {
+            _configuration = configuration;
+            _outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(ModelDebug));
+            Directory.CreateDirectory(_outputDirectory);
+        }
+
+        public int Index => 1;
+
+        public string Name => "Model Debug";
+
+        public string Description => "Model Debug";
+
+        public async Task RunAsync()
+        {
+            // Create Pipeline
+            // var pipeline = InstaFlowPipeline.CreatePipeline("D:\\Repositories\\Instaflow-onnx");
+            // var pipeline = LatentConsistencyPipeline.CreatePipeline("D:\\Repositories\\LCM_Dreamshaper_v7-onnx");
+            // var pipeline = LatentConsistencyXLPipeline.CreatePipeline("D:\\Repositories\\Latent-Consistency-xl-Olive-Onnx");
+            // var pipeline = StableDiffusionPipeline.CreatePipeline("D:\\Repositories\\stable-diffusion-v1-5");
+            var pipeline = StableDiffusionXLPipeline.CreatePipeline("D:\\Repositories\\Hyper-SD-onnx");
+
+            // Prompt
+            var promptOptions = new PromptOptions
+            {
+                Prompt = "a photo of a cat drinking at a bar with a penguin"
+            };
+
+            // Scheduler
+            var schedulerOptions = pipeline.DefaultSchedulerOptions with
+            {
+                InferenceSteps = 1,
+                GuidanceScale = 0,
+                SchedulerType = SchedulerType.DDIM,
+                Timesteps = new List<int> { 800 }
+            };
+
+            // Run pipeline
+            var result = await pipeline.RunAsync(promptOptions, schedulerOptions, progressCallback: OutputHelpers.ProgressCallback);
+
+            // Create Image from Tensor result
+            var image = new OnnxImage(result);
+
+            // Save Image File
+            await image.SaveAsync(Path.Combine(_outputDirectory, $"{pipeline.GetType().Name}-{schedulerOptions.Seed}.png"));
+
+            //Unload
+            await pipeline.UnloadAsync();
+        }
+    }
+}
diff --git a/OnnxStack.StableDiffusion/Config/SchedulerOptions.cs b/OnnxStack.StableDiffusion/Config/SchedulerOptions.cs
@@ -78,6 +78,7 @@ public record SchedulerOptions
         public PredictionType PredictionType { get; set; } = PredictionType.Epsilon;
         public AlphaTransformType AlphaTransformType { get; set; } = AlphaTransformType.Cosine;
         public float MaximumBeta { get; set; } = 0.999f;
+        public List<int> Timesteps { get; set; }
 
         public int OriginalInferenceSteps { get; set; } = 50;
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/InstaFlow/ControlNetDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/InstaFlow/ControlNetDiffuser.cs
@@ -164,6 +164,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/InstaFlow/TextDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/InstaFlow/TextDiffuser.cs
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
 using OnnxStack.Core.Model;
 using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
@@ -38,6 +39,9 @@ public TextDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoEn
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/ControlNetDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/ControlNetDiffuser.cs
@@ -160,6 +160,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/ControlNetImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/ControlNetImageDiffuser.cs
@@ -45,6 +45,9 @@ public ControlNetImageDiffuser(ControlNetModel controlNet, UNetConditionModel un
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/ImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/ImageDiffuser.cs
@@ -44,6 +44,9 @@ public ImageDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoE
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/InpaintLegacyDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/InpaintLegacyDiffuser.cs
@@ -47,6 +47,9 @@ public InpaintLegacyDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecode
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/TextDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistency/TextDiffuser.cs
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
 using OnnxStack.Core.Model;
 using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
@@ -38,6 +39,9 @@ public TextDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoEn
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/ControlNetDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/ControlNetDiffuser.cs
@@ -166,6 +166,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/ControlNetImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/ControlNetImageDiffuser.cs
@@ -45,6 +45,9 @@ public ControlNetImageDiffuser(ControlNetModel controlNet, UNetConditionModel un
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/ImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/ImageDiffuser.cs
@@ -45,6 +45,9 @@ public ImageDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoE
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             // Image2Image we narrow step the range by the Strength
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/InpaintLegacyDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/InpaintLegacyDiffuser.cs
@@ -141,6 +141,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/TextDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/LatentConsistencyXL/TextDiffuser.cs
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
 using OnnxStack.Core.Model;
 using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
@@ -38,6 +39,9 @@ public TextDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoEn
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/ControlNetDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/ControlNetDiffuser.cs
@@ -157,6 +157,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/ControlNetImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/ControlNetImageDiffuser.cs
@@ -45,6 +45,9 @@ public ControlNetImageDiffuser(ControlNetModel controlNet, UNetConditionModel un
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/ImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/ImageDiffuser.cs
@@ -44,6 +44,9 @@ public ImageDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoE
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             // Image2Image we narrow step the range by the Strength
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/InpaintDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/InpaintDiffuser.cs
@@ -252,6 +252,9 @@ private async Task<DenseTensor<float>> PrepareImageMask(PromptOptions promptOpti
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/InpaintLegacyDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/InpaintLegacyDiffuser.cs
@@ -135,6 +135,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/TextDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusion/TextDiffuser.cs
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
 using OnnxStack.Core.Model;
 using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
@@ -39,6 +40,9 @@ public TextDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoEn
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/ControlNetDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/ControlNetDiffuser.cs
@@ -167,6 +167,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/ControlNetImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/ControlNetImageDiffuser.cs
@@ -46,6 +46,9 @@ public ControlNetImageDiffuser(ControlNetModel controlNet, UNetConditionModel un
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/ImageDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/ImageDiffuser.cs
@@ -45,6 +45,9 @@ public ImageDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoE
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             // Image2Image we narrow step the range by the Strength
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/InpaintLegacyDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/InpaintLegacyDiffuser.cs
@@ -143,6 +143,9 @@ public override async Task<DenseTensor<float>> DiffuseAsync(PromptOptions prompt
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
             var start = Math.Max(options.InferenceSteps - inittimestep, 0);
             return scheduler.Timesteps.Skip(start).ToList();

diff --git a/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/TextDiffuser.cs b/OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/TextDiffuser.cs
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core;
 using OnnxStack.Core.Model;
 using OnnxStack.StableDiffusion.Common;
 using OnnxStack.StableDiffusion.Config;
@@ -39,6 +40,9 @@ public TextDiffuser(UNetConditionModel unet, AutoEncoderModel vaeDecoder, AutoEn
         /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
+            if (!options.Timesteps.IsNullOrEmpty())
+                return options.Timesteps;
+
             return scheduler.Timesteps;
         }
 

diff --git a/OnnxStack.StableDiffusion/Schedulers/SchedulerBase.cs b/OnnxStack.StableDiffusion/Schedulers/SchedulerBase.cs
@@ -174,7 +174,7 @@ protected virtual float[] GetTimesteps()
             }
             else if (Options.TimestepSpacing == TimestepSpacingType.Trailing)
             {
-                var stepRatio = Options.TrainTimesteps / (Options.InferenceSteps - 1);
+                var stepRatio = Options.TrainTimesteps / Math.Max(1, (Options.InferenceSteps - 1));
                 return Enumerable.Range(0, Options.TrainTimesteps)
                     .Where((number, index) => index % stepRatio == 0)
                     .Select(x => (float)x)