Update requirements and others

Author: bmaltais

README.md

@@ -483,6 +483,7 @@ If you come across a `FileNotFoundError`, it is likely due to an installation is
 * 2023/07/29 (v21.8.6)
   - Merge latest sd-scripts updates.
   - Allow DB training on SDXL models. Unsupported but appear to work.
+  - Fix finetuning latent caching issue when doing SDXL models in fp16
 * 2023/07/27 (v21.8.5)
   - Backrev the LyCORIS module version due to bug reports.
 * 2023/07/27 (v21.8.4)

finetune_gui.py

@@ -450,6 +450,9 @@ def train_model(
         #     run_cmd += f' --flip_aug'
         if full_path:
             run_cmd += f' --full_path'
+        if sdxl_no_half_vae:
+            log.info('Using mixed_precision = no because no half vae is selected...')
+            run_cmd += f' --mixed_precision="no"'
 
         log.info(run_cmd)
 

requirements.txt

@@ -10,7 +10,7 @@ ftfy==6.1.1
 gradio==3.36.1
 huggingface-hub==0.15.1
 lion-pytorch==0.0.6
-lycoris_lora==1.8.0
+lycoris_lora==1.8.3
 # for loading Diffusers' SDXL
 invisible-watermark==0.2.0
 # open clip for SDXL

requirements_windows_torch2.txt

@@ -1,5 +1,6 @@
 torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118 # no_verify
 xformers==0.0.20
+# bitsandbytes==0.35.0
 https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.40.1.post1-py3-none-win_amd64.whl # no_verify
 tensorboard==2.12.3 tensorflow==2.12.0
 -r requirements.txt

tools/extract_locon.py

@@ -1,9 +1,8 @@
-#
-# From: https://raw.githubusercontent.com/KohakuBlueleaf/LoCon/main/extract_locon.py
-#
-
+import os, sys
+sys.path.insert(0, os.getcwd())
 import argparse
 
+
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -29,11 +28,15 @@ def get_args():
     parser.add_argument(
         "--mode", 
         help=(
-            'extraction mode, can be "fixed", "threshold", "ratio", "percentile". '
+            'extraction mode, can be "fixed", "threshold", "ratio", "quantile". '
             'If not "fixed", network_dim and conv_dim will be ignored'
         ),
         default='fixed', type=str
     )
+    parser.add_argument(
+        "--safetensors", help='use safetensors to save locon model',
+        default=True, action="store_true"
+    )
     parser.add_argument(
         "--linear_dim", help="network dim for linear layer in fixed mode",
         default=1, type=int
@@ -59,20 +62,34 @@ def get_args():
         default=0., type=float
     )
     parser.add_argument(
-        "--linear_percentile", help="singular value percentile for linear layer percentile mode",
+        "--linear_quantile", help="singular value quantile for linear layer quantile mode",
         default=1., type=float
     )
     parser.add_argument(
-        "--conv_percentile", help="singular value percentile for conv layer percentile mode",
+        "--conv_quantile", help="singular value quantile for conv layer quantile mode",
         default=1., type=float
     )
+    parser.add_argument(
+        "--use_sparse_bias", help="enable sparse bias",
+        default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--sparsity", help="sparsity for sparse bias",
+        default=0.98, type=float
+    )
+    parser.add_argument(
+        "--disable_cp", help="don't use cp decomposition",
+        default=False, action="store_true"
+    )
     return parser.parse_args()
 ARGS = get_args()
 
-from locon.utils import extract_diff
-from locon.kohya_model_utils import load_models_from_stable_diffusion_checkpoint
+
+from lycoris.utils import extract_diff
+from lycoris.kohya.model_utils import load_models_from_stable_diffusion_checkpoint
 
 import torch
+from safetensors.torch import save_file
 
 
 def main():
@@ -84,22 +101,28 @@ def main():
         'fixed': args.linear_dim,
         'threshold': args.linear_threshold,
         'ratio': args.linear_ratio,
-        'percentile': args.linear_percentile,
+        'quantile': args.linear_quantile,
     }[args.mode]
     conv_mode_param = {
         'fixed': args.conv_dim,
         'threshold': args.conv_threshold,
         'ratio': args.conv_ratio,
-        'percentile': args.conv_percentile,
+        'quantile': args.conv_quantile,
     }[args.mode]
 
     state_dict = extract_diff(
         base, db,
         args.mode,
         linear_mode_param, conv_mode_param,
-        args.device
+        args.device, 
+        args.use_sparse_bias, args.sparsity,
+        not args.disable_cp
     )
-    torch.save(state_dict, args.output_name)
+    
+    if args.safetensors:
+        save_file(state_dict, args.output_name)
+    else:
+        torch.save(state_dict, args.output_name)
 
 
 if __name__ == '__main__':

tools/merge_lycoris.py

@@ -37,8 +37,8 @@ def get_args():
 ARGS = get_args()
 
 
-from lycoris_utils import merge
-from lycoris.kohya_model_utils import (
+from lycoris.utils import merge
+from lycoris.kohya.model_utils import (
     load_models_from_stable_diffusion_checkpoint,
     save_stable_diffusion_checkpoint,
     load_file