Do not re-encode the exact same image twice

Author: stduhpf

stable-diffusion.cpp

@@ -49,8 +49,7 @@ const char* sampling_methods_str[] = {
     "iPNDM_v",
     "LCM",
     "DDIM \"trailing\"",
-    "TCD"
-};
+    "TCD"};
 
 /*================================================== Helper Functions ================================================*/
 
@@ -683,7 +682,7 @@ class StableDiffusionGGML {
             float curr_multiplier        = kv.second;
             lora_state_diff[lora_name] -= curr_multiplier;
         }
-        
+
         size_t rm = lora_state_diff.size() - lora_state.size();
         if (rm != 0) {
             LOG_INFO("Attempting to apply %lu LoRAs (removing %lu applied LoRAs)", lora_state.size(), rm);
@@ -806,7 +805,6 @@ class StableDiffusionGGML {
                         float skip_layer_start       = 0.01,
                         float skip_layer_end         = 0.2,
                         ggml_tensor* noise_mask      = nullptr) {
-
         // TODO (Pix2Pix): separate image guidance params (right now it's reusing distilled guidance)
 
         float img_cfg_scale = guidance;
@@ -834,7 +832,7 @@ class StableDiffusionGGML {
 
         bool has_unconditioned = cfg_scale != 1.0 && uncond.c_crossattn != NULL;
         bool has_img_guidance  = version == VERSION_INSTRUCT_PIX2PIX && cfg_scale != img_cfg_scale;
-        has_unconditioned = has_unconditioned || has_img_guidance;
+        has_unconditioned      = has_unconditioned || has_img_guidance;
         bool has_skiplayer     = slg_scale != 0.0 && skip_layers.size() > 0;
 
         // denoise wrapper
@@ -988,7 +986,7 @@ class StableDiffusionGGML {
                         int64_t i3  = i / out_cond->ne[0] * out_cond->ne[1] * out_cond->ne[2];
                         float scale = min_cfg + (cfg_scale - min_cfg) * (i3 * 1.0f / ne3);
                     } else {
-                        if(has_img_guidance){
+                        if (has_img_guidance) {
                             latent_result = negative_data[i] + img_cfg_scale * (img_cond_data[i] - negative_data[i]) + cfg_scale * (positive_data[i] - img_cond_data[i]);
                         } else {
                             latent_result = negative_data[i] + cfg_scale * (positive_data[i] - negative_data[i]);
@@ -1393,7 +1391,7 @@ sd_image_t* generate_image(sd_ctx_t* sd_ctx,
                                                                            sd_ctx->sd->diffusion_model->get_adm_in_channels());
 
     SDCondition uncond;
-    if (cfg_scale != 1.0 || sd_ctx->sd->version == VERSION_INSTRUCT_PIX2PIX && cfg_scale!=guidance) {
+    if (cfg_scale != 1.0 || sd_ctx->sd->version == VERSION_INSTRUCT_PIX2PIX && cfg_scale != guidance) {
         bool force_zero_embeddings = false;
         if (sd_version_is_sdxl(sd_ctx->sd->version) && negative_prompt.size() == 0) {
             force_zero_embeddings = true;
@@ -1739,6 +1737,14 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx,
 
     sd_image_to_tensor(init_image.data, init_img);
 
+    ggml_tensor* init_latent = NULL;
+    if (!sd_ctx->sd->use_tiny_autoencoder) {
+        ggml_tensor* moments = sd_ctx->sd->encode_first_stage(work_ctx, init_img);
+        init_latent          = sd_ctx->sd->get_first_stage_encoding(work_ctx, moments);
+    } else {
+        init_latent = sd_ctx->sd->encode_first_stage(work_ctx, init_img);
+    }
+
     ggml_tensor* masked_image;
 
     if (sd_version_is_inpaint(sd_ctx->sd->version)) {
@@ -1786,12 +1792,7 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx,
         }
     } else if (sd_ctx->sd->version == VERSION_INSTRUCT_PIX2PIX) {
         // Not actually masked, we're just highjacking the masked_image variable since it will be used the same way
-        if (!sd_ctx->sd->use_tiny_autoencoder) {
-            ggml_tensor* moments = sd_ctx->sd->encode_first_stage(work_ctx, init_img);
-            masked_image         = sd_ctx->sd->get_first_stage_encoding(work_ctx, moments);
-        } else {
-            masked_image = sd_ctx->sd->encode_first_stage(work_ctx, init_img);
-        }
+        masked_image = init_latent;
     } else {
         // LOG_WARN("Inpainting with a base model is not great");
         masked_image = ggml_new_tensor_4d(work_ctx, GGML_TYPE_F32, width / 8, height / 8, 1, 1);
@@ -1805,14 +1806,6 @@ sd_image_t* img2img(sd_ctx_t* sd_ctx,
         }
     }
 
-    ggml_tensor* init_latent = NULL;
-    if (!sd_ctx->sd->use_tiny_autoencoder) {
-        ggml_tensor* moments = sd_ctx->sd->encode_first_stage(work_ctx, init_img);
-        init_latent          = sd_ctx->sd->get_first_stage_encoding(work_ctx, moments);
-    } else {
-        init_latent = sd_ctx->sd->encode_first_stage(work_ctx, init_img);
-    }
-
     print_ggml_tensor(init_latent, true);
     size_t t1 = ggml_time_ms();
     LOG_INFO("encode_first_stage completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);