From bd8fe8a5f1b1fab3f90ec166ae5950b16682089f Mon Sep 17 00:00:00 2001 From: Nerogar Date: Fri, 13 Oct 2023 21:56:53 +0200 Subject: [PATCH] code formatting --- .gitignore | 4 +- modules/ui/OptimizerParamsWindow.py | 71 +++++++++++++++-------------- modules/util/create.py | 48 +++++++++---------- 3 files changed, 63 insertions(+), 60 deletions(-) diff --git a/.gitignore b/.gitignore index b3247aea..45c73c31 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,9 @@ /models* /training_concepts /training_samples +/training_user_settings /external debug.py train.bat *.pyc - *.bak - -training_user_settings/ diff --git a/modules/ui/OptimizerParamsWindow.py b/modules/ui/OptimizerParamsWindow.py index 04c075bc..bf805267 100644 --- a/modules/ui/OptimizerParamsWindow.py +++ b/modules/ui/OptimizerParamsWindow.py @@ -1,10 +1,14 @@ +import json +import math +import os +from tkinter import TclError + import customtkinter as ctk -from modules.util.ui import components + from modules.util.args.TrainArgs import TrainArgs from modules.util.enum.Optimizer import Optimizer -import math -import json -import os +from modules.util.ui import components + class UserPreferenceUtility: def __init__(self, file_path="training_user_settings/optimizer_prefs.json"): @@ -13,7 +17,6 @@ def __init__(self, file_path="training_user_settings/optimizer_prefs.json"): if not os.path.exists(directory): os.mkdir(directory) - def load_preferences(self, optimizer_name): if os.path.exists(self.file_path): with open(self.file_path, 'r') as f: @@ -36,6 +39,7 @@ def save_preference(self, optimizer_name, key, value): with open(self.file_path, 'w') as f: json.dump(prefs, f, indent=4) + class OptimizerParamsWindow(ctk.CTkToplevel): def __init__(self, parent, ui_state, *args, **kwargs): self.pref_util = UserPreferenceUtility() @@ -202,7 +206,7 @@ def __init__(self, parent, ui_state, *args, **kwargs): "optimizer_eps": 1e-8, "optimizer_weight_decay": 0, "optimizer_decouple": True, - "optimizer_use_bias_correction": False, + "optimizer_use_bias_correction": False, "optimizer_safeguard_warmup": False, "optimizer_d0": 1e-6, "optimizer_d_coef": 1.0, @@ -288,7 +292,7 @@ def __init__(self, parent, ui_state, *args, **kwargs): "optimizer_foreach": False, "optimizer_maximize": False, "optimizer_differentiable": False - }, + }, "LION": { "optimizer_beta1": 0.9, "optimizer_beta2": 0.99, @@ -296,7 +300,7 @@ def __init__(self, parent, ui_state, *args, **kwargs): "optimizer_use_triton": False }, } - + self.grid_rowconfigure(0, weight=1) self.grid_rowconfigure(1, weight=0) self.grid_columnconfigure(0, weight=1) @@ -309,17 +313,18 @@ def __init__(self, parent, ui_state, *args, **kwargs): self.frame.grid_columnconfigure(2, minsize=50) self.frame.grid_columnconfigure(3, weight=0) self.frame.grid_columnconfigure(4, weight=1) - + components.button(self, 1, 0, "ok", self.__ok) self.button = None - self.main_frame(self.frame) - + self.main_frame(self.frame) + def __ok(self): self.destroy() - + def create_dynamic_ui(self, selected_optimizer, master, components, ui_state, defaults=False): - #Lookup for the title and tooltip for a key + # Lookup for the title and tooltip for a key + # @formatter:off KEY_DETAIL_MAP = { 'optimizer_adam_w_mode': {'title': 'Adam W Mode', 'tooltip': 'Whether to use weight decay correction for Adam optimizer.', 'type': 'bool'}, 'optimizer_alpha': {'title': 'Alpha', 'tooltip': 'Smoothing parameter for RMSprop and others.', 'type': 'float'}, @@ -364,12 +369,13 @@ def create_dynamic_ui(self, selected_optimizer, master, components, ui_state, de 'optimizer_warmup_init': {'title': 'Warmup Initialization', 'tooltip': 'Whether to warm-up the optimizer initialization.', 'type': 'bool'}, 'optimizer_weight_decay': {'title': 'Weight Decay', 'tooltip': 'Regularization to prevent overfitting.', 'type': 'float'}, } - - + # @formatter:on + if not self.winfo_exists(): # check if this window isn't open return - optimizer_keys = list(self.OPTIMIZER_KEY_MAP[selected_optimizer].keys()) # Extract the keys for the selected optimizer + # Extract the keys for the selected optimizer + optimizer_keys = list(self.OPTIMIZER_KEY_MAP[selected_optimizer].keys()) for idx, key in enumerate(optimizer_keys): arg_info = KEY_DETAIL_MAP[key] @@ -378,7 +384,7 @@ def create_dynamic_ui(self, selected_optimizer, master, components, ui_state, de type = arg_info['type'] row = math.floor(idx / 2) + 1 - col = 3 * (idx % 2) + col = 3 * (idx % 2) components.label(master, row, col, title, tooltip=tooltip) override_value = None @@ -390,32 +396,33 @@ def create_dynamic_ui(self, selected_optimizer, master, components, ui_state, de override_value = self.OPTIMIZER_KEY_MAP[selected_optimizer][key] if type != 'bool': - entry_widget = components.entry(master, row, col+1, ui_state, key, override_value=override_value) + entry_widget = components.entry(master, row, col + 1, ui_state, key, override_value=override_value) entry_widget.bind("", lambda event, opt=selected_optimizer, k=key: self.update_user_pref(opt, k, ui_state.vars[k].get())) else: - switch_widget = components.switch(master, row, col+1, ui_state, key, override_value=override_value) + switch_widget = components.switch(master, row, col + 1, ui_state, key, override_value=override_value) switch_widget.configure(command=lambda opt=selected_optimizer, k=key: self.update_user_pref(opt, k, ui_state.vars[k].get())) def update_user_pref(self, optimizer, key, value): self.pref_util.save_preference(optimizer, key, value) - - def main_frame(self, master): - + # Optimizer - components.label(master, 0, 0, "Optimizer", tooltip="The type of optimizer") + components.label(master, 0, 0, "Optimizer", + tooltip="The type of optimizer") components.options(master, 0, 1, [str(x) for x in list(Optimizer)], self.ui_state, "optimizer") - + # Defaults Button - components.label(master, 0, 0, "Optimizer Defaults", tooltip="Load default settings for the selected optimizer") - components.button(self.frame, 0, 4, "Load Defaults", self.load_defaults, tooltip="Load default settings for the selected optimizer") + components.label(master, 0, 0, "Optimizer Defaults", + tooltip="Load default settings for the selected optimizer") + components.button(self.frame, 0, 4, "Load Defaults", self.load_defaults, + tooltip="Load default settings for the selected optimizer") selected_optimizer = self.ui_state.vars['optimizer'].get() - + self.ui_state.vars['optimizer'].trace_add('write', self.on_optimizer_change) self.create_dynamic_ui(selected_optimizer, master, components, self.ui_state) - + def on_optimizer_change(self, *args): selected_optimizer = self.ui_state.vars['optimizer'].get() user_prefs = self.pref_util.load_preferences(selected_optimizer) @@ -428,14 +435,14 @@ def on_optimizer_change(self, *args): if value_to_set is None: value_to_set = "None" - + self.ui_state.vars[key].set(value_to_set) if not self.winfo_exists(): # check if this window isn't open return self.clear_dynamic_ui(self.frame) self.create_dynamic_ui(selected_optimizer, self.frame, components, self.ui_state) - + def load_defaults(self): if not self.winfo_exists(): # check if this window isn't open return @@ -443,13 +450,11 @@ def load_defaults(self): self.clear_dynamic_ui(self.frame) self.create_dynamic_ui(selected_optimizer, self.frame, components, self.ui_state, defaults=True) - def clear_dynamic_ui(self, master): try: for widget in master.winfo_children(): grid_info = widget.grid_info() if int(grid_info["row"]) >= 1: widget.destroy() - except _tkinter.TclError as e: + except TclError as e: pass - diff --git a/modules/util/create.py b/modules/util/create.py index e7088f9d..0d220eb9 100644 --- a/modules/util/create.py +++ b/modules/util/create.py @@ -231,7 +231,7 @@ def create_optimizer( nesterov=args.optimizer_nesterov or False, foreach=args.optimizer_foreach or False, maximize=args.optimizer_maximize or False, - differentiable=args.optimizer_differentiable or False + differentiable=args.optimizer_differentiable or False, ) # SGD_8BIT Optimizer @@ -243,7 +243,7 @@ def create_optimizer( momentum=args.optimizer_momentum or 0, dampening=args.optimizer_dampening or 0, weight_decay=args.optimizer_weight_decay or 0, - nesterov=args.optimizer_nesterov or False + nesterov=args.optimizer_nesterov or False, ) # ADAM Optimizer @@ -259,7 +259,7 @@ def create_optimizer( maximize=args.optimizer_maximize or False, capturable=args.optimizer_capturable or False, differentiable=args.optimizer_differentiable or False, - fused=args.optimizer_fused or False + fused=args.optimizer_fused or False, ) # ADAMW Optimizer @@ -275,7 +275,7 @@ def create_optimizer( maximize=args.optimizer_maximize or False, capturable=args.optimizer_capturable or False, differentiable=args.optimizer_differentiable or False, - fused=args.optimizer_fused or False + fused=args.optimizer_fused or False, ) # ADAM_8BIT Optimizer @@ -289,7 +289,7 @@ def create_optimizer( min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, block_wise=args.optimizer_block_wise or True, - is_paged=args.optimizer_is_paged or False + is_paged=args.optimizer_is_paged or False, ) # ADAMW_8BIT Optimizer @@ -303,7 +303,7 @@ def create_optimizer( min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, block_wise=args.optimizer_block_wise or True, - is_paged=args.optimizer_is_paged or False + is_paged=args.optimizer_is_paged or False, ) # ADAGRAD Optimizer @@ -315,7 +315,7 @@ def create_optimizer( weight_decay=args.optimizer_weight_decay or 0, eps=args.optimizer_eps or 1e-10, lr_decay=args.optimizer_lr_decay or 0, - initial_accumulator_value=args.optimizer_initial_accumulator_value or 0 + initial_accumulator_value=args.optimizer_initial_accumulator_value or 0, ) # ADAGRAD_8BIT Optimizer @@ -330,7 +330,7 @@ def create_optimizer( initial_accumulator_value=args.optimizer_initial_accumulator_value or 0, min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, - block_wise=args.optimizer_block_wise or True + block_wise=args.optimizer_block_wise or True, ) # RMSPROP Optimizer @@ -343,7 +343,7 @@ def create_optimizer( eps=args.optimizer_eps or 1e-8, alpha=args.optimizer_alpha or 0.99, momentum=args.optimizer_momentum or 0, - centered=args.optimizer_centered or False + centered=args.optimizer_centered or False, ) # RMSPROP_8BIT Optimizer @@ -359,7 +359,7 @@ def create_optimizer( centered=args.optimizer_centered or False, min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, - block_wise=args.optimizer_block_wise or True + block_wise=args.optimizer_block_wise or True, ) # LION Optimizer @@ -370,7 +370,7 @@ def create_optimizer( lr=args.learning_rate, betas=(args.optimizer_beta1 or 0.9, args.optimizer_beta2 or 0.99), weight_decay=args.optimizer_weight_decay or 0, - use_triton=args.optimizer_use_triton or False + use_triton=args.optimizer_use_triton or False, ) # LARS Optimizer @@ -383,7 +383,7 @@ def create_optimizer( momentum=args.optimizer_momentum or 0, dampening=args.optimizer_dampening or 0, nesterov=args.optimizer_nesterov or False, - max_unorm=args.optimizer_max_unorm or 0.02 + max_unorm=args.optimizer_max_unorm or 0.02, ) # LARS_8BIT Optimizer @@ -398,7 +398,7 @@ def create_optimizer( nesterov=args.optimizer_nesterov or False, min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, - max_unorm=args.optimizer_max_unorm or 0.02 + max_unorm=args.optimizer_max_unorm or 0.02, ) # LAMB Optimizer @@ -414,7 +414,7 @@ def create_optimizer( adam_w_mode=args.optimizer_adam_w_mode or True, percentile_clipping=args.optimizer_percentile_clipping or 100, block_wise=args.optimizer_block_wise or False, - max_unorm=args.optimizer_max_unorm or 1.0 + max_unorm=args.optimizer_max_unorm or 1.0, ) # LAMB_8BIT Optimizer @@ -431,7 +431,7 @@ def create_optimizer( min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, block_wise=args.optimizer_block_wise or False, - max_unorm=args.optimizer_max_unorm or 1.0 + max_unorm=args.optimizer_max_unorm or 1.0, ) # LION_8BIT Optimizer @@ -445,7 +445,7 @@ def create_optimizer( min_8bit_size=args.optimizer_min_8bit_size or 4096, percentile_clipping=args.optimizer_percentile_clipping or 100, block_wise=args.optimizer_block_wise or True, - is_paged=args.optimizer_is_paged or False + is_paged=args.optimizer_is_paged or False, ) # DADAPT_SGD Optimizer @@ -459,7 +459,7 @@ def create_optimizer( log_every=args.optimizer_log_every or 0, d0=args.optimizer_d0 or 1e-6, growth_rate=args.optimizer_growth_rate or float('inf'), - fsdp_in_use=args.optimizer_fsdp_in_use or False + fsdp_in_use=args.optimizer_fsdp_in_use or False, ) # DADAPT_ADAM Optimizer @@ -476,7 +476,7 @@ def create_optimizer( use_bias_correction=args.optimizer_use_bias_correction or False, d0=args.optimizer_d0 or 1e-6, growth_rate=args.optimizer_growth_rate or float('inf'), - fsdp_in_use=args.optimizer_fsdp_in_use or False + fsdp_in_use=args.optimizer_fsdp_in_use or False, ) # DADAPT_ADAN Optimizer @@ -491,7 +491,7 @@ def create_optimizer( no_prox=args.optimizer_no_prox or False, log_every=args.optimizer_log_every or 0, d0=args.optimizer_d0 or 1e-6, - growth_rate=args.optimizer_growth_rate or float('inf') + growth_rate=args.optimizer_growth_rate or float('inf'), ) # DADAPT_ADA_GRAD Optimizer @@ -505,7 +505,7 @@ def create_optimizer( weight_decay=args.optimizer_weight_decay or 0.0, eps=args.optimizer_eps or 0.0, d0=args.optimizer_d0 or 1e-6, - growth_rate=args.optimizer_growth_rate or float('inf') + growth_rate=args.optimizer_growth_rate or float('inf'), ) # DADAPT_LION Optimizer @@ -518,7 +518,7 @@ def create_optimizer( weight_decay=args.optimizer_weight_decay or 0.0, log_every=args.optimizer_log_every or 0, d0=args.optimizer_d0 or 1e-6, - fsdp_in_use=args.optimizer_fsdp_in_use or False + fsdp_in_use=args.optimizer_fsdp_in_use or False, ) # PRODIGY Optimizer @@ -537,7 +537,7 @@ def create_optimizer( d0=args.optimizer_d0 or 1e-6, d_coef=args.optimizer_d_coef or 1.0, growth_rate=args.optimizer_growth_rate or float('inf'), - fsdp_in_use=args.optimizer_fsdp_in_use or False + fsdp_in_use=args.optimizer_fsdp_in_use or False, ) # ADAFactor Optimizer @@ -553,7 +553,7 @@ def create_optimizer( weight_decay=args.optimizer_weight_decay or 0.0, scale_parameter=args.optimizer_scale_parameter or True, relative_step=args.optimizer_relative_step or True, - warmup_init=args.optimizer_warmup_init or False + warmup_init=args.optimizer_warmup_init or False, ) else: optimizer = Adafactor( @@ -566,7 +566,7 @@ def create_optimizer( weight_decay=args.optimizer_weight_decay or 0.0, scale_parameter=args.optimizer_scale_parameter or True, relative_step=args.optimizer_relative_step or True, - warmup_init=args.optimizer_warmup_init or False + warmup_init=args.optimizer_warmup_init or False, ) if state_dict is not None: