Merge pull request #3975 from aria1th/force-push-patch-13

from statistics import stdev, mean

optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"}

class HypernetworkModule(torch.nn.Module):

    multiplier = 1.0

    activation_dict = {

        self.use_dropout = use_dropout

        self.activate_output = activate_output

        self.last_layer_dropout = kwargs['last_layer_dropout'] if 'last_layer_dropout' in kwargs else True

        self.optimizer_name = None

        self.optimizer_state_dict = None

        for size in enable_sizes or []:

            self.layers[size] = (

    def save(self, filename):

        state_dict = {}

        optimizer_saved_dict = {}

        for k, v in self.layers.items():

            state_dict[k] = (v[0].state_dict(), v[1].state_dict())

        state_dict['activate_output'] = self.activate_output

        state_dict['last_layer_dropout'] = self.last_layer_dropout

        if self.optimizer_name is not None:

            optimizer_saved_dict['optimizer_name'] = self.optimizer_name

        torch.save(state_dict, filename)

        if shared.opts.save_optimizer_state and self.optimizer_state_dict:

            optimizer_saved_dict['hash'] = sd_models.model_hash(filename)

            optimizer_saved_dict['optimizer_state_dict'] = self.optimizer_state_dict

            torch.save(optimizer_saved_dict, filename + '.optim')

    def load(self, filename):

        self.filename = filename

        print(f"Activate last layer is set to {self.activate_output}")

        self.last_layer_dropout = state_dict.get('last_layer_dropout', False)

        optimizer_saved_dict = torch.load(self.filename + '.optim', map_location = 'cpu') if os.path.exists(self.filename + '.optim') else {}

        self.optimizer_name = optimizer_saved_dict.get('optimizer_name', 'AdamW')

        print(f"Optimizer name is {self.optimizer_name}")

        if sd_models.model_hash(filename) == optimizer_saved_dict.get('hash', None):

            self.optimizer_state_dict = optimizer_saved_dict.get('optimizer_state_dict', None)

        else:

            self.optimizer_state_dict = None

        if self.optimizer_state_dict:

            print("Loaded existing optimizer from checkpoint")

        else:

            print("No saved optimizer exists in checkpoint")

        for size, sd in state_dict.items():

            if type(size) == int:

                self.layers[size] = (

        name = os.path.splitext(os.path.basename(filename))[0]

        # Prevent a hypothetical "None.pt" from being listed.

        if name != "None":

            res[name] = filename

            res[name + f"({sd_models.model_hash(filename)})"] = filename

    return res

    shared.state.textinfo = "Initializing hypernetwork training..."

    shared.state.job_count = steps

    hypernetwork_name = hypernetwork_name.rsplit('(', 1)[0]

    filename = os.path.join(shared.cmd_opts.hypernetwork_dir, f'{hypernetwork_name}.pt')

    log_directory = os.path.join(log_directory, datetime.datetime.now().strftime("%Y-%m-%d"), hypernetwork_name)

    weights = hypernetwork.weights()

    for weight in weights:

        weight.requires_grad = True

    # if optimizer == "AdamW": or else Adam / AdamW / SGD, etc...

    optimizer = torch.optim.AdamW(weights, lr=scheduler.learn_rate)

    # Here we use optimizer from saved HN, or we can specify as UI option.

    if (optimizer_name := hypernetwork.optimizer_name) in optimizer_dict:

        optimizer = optimizer_dict[hypernetwork.optimizer_name](params=weights, lr=scheduler.learn_rate)

    else:

        print(f"Optimizer type {optimizer_name} is not defined!")

        optimizer = torch.optim.AdamW(params=weights, lr=scheduler.learn_rate)

        optimizer_name = 'AdamW'

    if hypernetwork.optimizer_state_dict:  # This line must be changed if Optimizer type can be different from saved optimizer.

        try:

            optimizer.load_state_dict(hypernetwork.optimizer_state_dict)

        except RuntimeError as e:

            print("Cannot resume from saved optimizer!")

            print(e)

    steps_without_grad = 0

            # Before saving, change name to match current checkpoint.

            hypernetwork_name_every = f'{hypernetwork_name}-{steps_done}'

            last_saved_file = os.path.join(hypernetwork_dir, f'{hypernetwork_name_every}.pt')

            hypernetwork.optimizer_name = optimizer_name

            if shared.opts.save_optimizer_state:

                hypernetwork.optimizer_state_dict = optimizer.state_dict()

            save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, last_saved_file)

            hypernetwork.optimizer_state_dict = None  # dereference it after saving, to save memory.

        textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, len(ds), {

            "loss": f"{previous_mean_loss:.7f}",

    report_statistics(loss_dict)

    filename = os.path.join(shared.cmd_opts.hypernetwork_dir, f'{hypernetwork_name}.pt')

    hypernetwork.optimizer_name = optimizer_name

    if shared.opts.save_optimizer_state:

        hypernetwork.optimizer_state_dict = optimizer.state_dict()

    save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, filename)

    del optimizer

    hypernetwork.optimizer_state_dict = None  # dereference it after saving, to save memory.

    return hypernetwork, filename

def save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, filename):

from modules.hypernetworks import hypernetwork

not_available = ["hardswish", "multiheadattention"]

keys = ["linear"] + list(x for x in hypernetwork.HypernetworkModule.activation_dict.keys() if x not in not_available)

keys = list(x for x in hypernetwork.HypernetworkModule.activation_dict.keys() if x not in not_available)

def create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure=None, activation_func=None, weight_init=None, add_layer_norm=False, use_dropout=False):

    # Remove illegal characters from name.

options_templates.update(options_section(('training', "Training"), {

    "unload_models_when_training": OptionInfo(False, "Move VAE and CLIP to RAM when training if possible. Saves VRAM."),

    "save_optimizer_state": OptionInfo(False, "Saves Optimizer state as separate *.optim file. Training can be resumed with HN itself and matching optim file."),

    "dataset_filename_word_regex": OptionInfo("", "Filename word regex"),

    "dataset_filename_join_string": OptionInfo(" ", "Filename join string"),

    "training_image_repeats_per_epoch": OptionInfo(1, "Number of repeats for a single input image per epoch; used only for displaying epoch number", gr.Number, {"precision": 0}),