apply

from statistics import stdev, mean

optimizer_dict = {optim_name : cls_obj for optim_name, cls_obj in inspect.getmembers(torch.optim, inspect.isclass) if optim_name != "Optimizer"}

class HypernetworkModule(torch.nn.Module):

    multiplier = 1.0

    activation_dict = {

        self.use_dropout = use_dropout

        self.activate_output = activate_output

        self.last_layer_dropout = kwargs['last_layer_dropout'] if 'last_layer_dropout' in kwargs else True

        self.optimizer_name = None

        self.optimizer_state_dict = None

        for size in enable_sizes or []:

            self.layers[size] = (

    def save(self, filename):

        state_dict = {}

        optimizer_saved_dict = {}

        for k, v in self.layers.items():

            state_dict[k] = (v[0].state_dict(), v[1].state_dict())

        state_dict['activate_output'] = self.activate_output

        state_dict['last_layer_dropout'] = self.last_layer_dropout

        if self.optimizer_name is not None:

            optimizer_saved_dict['optimizer_name'] = self.optimizer_name

        torch.save(state_dict, filename)

        if self.optimizer_state_dict:

            optimizer_saved_dict['hash'] = sd_models.model_hash(filename)

            optimizer_saved_dict['optimizer_state_dict'] = self.optimizer_state_dict

            torch.save(optimizer_saved_dict, filename + '.optim')

    def load(self, filename):

        self.filename = filename

        print(f"Activate last layer is set to {self.activate_output}")

        self.last_layer_dropout = state_dict.get('last_layer_dropout', False)

        optimizer_saved_dict = torch.load(self.filename + '.optim', map_location = 'cpu') if os.path.exists(self.filename + '.optim') else {}

        self.optimizer_name = optimizer_saved_dict.get('optimizer_name', 'AdamW')

        print(f"Optimizer name is {self.optimizer_name}")

        if sd_models.model_hash(filename) == optimizer_saved_dict.get('hash', None):

            self.optimizer_state_dict = optimizer_saved_dict.get('optimizer_state_dict', None)

        else:

            self.optimizer_state_dict = None

        if self.optimizer_state_dict:

            print("Loaded existing optimizer from checkpoint")

        else:

            print("No saved optimizer exists in checkpoint")

        for size, sd in state_dict.items():

            if type(size) == int:

                self.layers[size] = (

        name = os.path.splitext(os.path.basename(filename))[0]

        # Prevent a hypothetical "None.pt" from being listed.

        if name != "None":

            res[name] = filename

            res[name + f"({sd_models.model_hash(filename)})"] = filename

    return res

    else:

        hypernetwork_dir = None

    hypernetwork_name = hypernetwork_name.rsplit('(', 1)[0]

    if create_image_every > 0:

        images_dir = os.path.join(log_directory, "images")

        os.makedirs(images_dir, exist_ok=True)

    weights = hypernetwork.weights()

    for weight in weights:

        weight.requires_grad = True

    # if optimizer == "AdamW": or else Adam / AdamW / SGD, etc...

    optimizer = torch.optim.AdamW(weights, lr=scheduler.learn_rate)

    # Here we use optimizer from saved HN, or we can specify as UI option.

    if (optimizer_name := hypernetwork.optimizer_name) in optimizer_dict:

        optimizer = optimizer_dict[hypernetwork.optimizer_name](params=weights, lr=scheduler.learn_rate)

    else:

        print(f"Optimizer type {optimizer_name} is not defined!")

        optimizer = torch.optim.AdamW(params=weights, lr=scheduler.learn_rate)

        optimizer_name = 'AdamW'

    if hypernetwork.optimizer_state_dict:  # This line must be changed if Optimizer type can be different from saved optimizer.

        try:

            optimizer.load_state_dict(hypernetwork.optimizer_state_dict)

        except RuntimeError as e:

            print("Cannot resume from saved optimizer!")

            print(e)

    steps_without_grad = 0

            # Before saving, change name to match current checkpoint.

            hypernetwork_name_every = f'{hypernetwork_name}-{steps_done}'

            last_saved_file = os.path.join(hypernetwork_dir, f'{hypernetwork_name_every}.pt')

            hypernetwork.optimizer_name = optimizer_name

            if shared.opts.save_optimizer_state:

                hypernetwork.optimizer_state_dict = optimizer.state_dict()

            save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, last_saved_file)

            hypernetwork.optimizer_state_dict = None  # dereference it after saving, to save memory.

        textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, len(ds), {

            "loss": f"{previous_mean_loss:.7f}",

    report_statistics(loss_dict)

    filename = os.path.join(shared.cmd_opts.hypernetwork_dir, f'{hypernetwork_name}.pt')

    hypernetwork.optimizer_name = optimizer_name

    if shared.opts.save_optimizer_state:

        hypernetwork.optimizer_state_dict = optimizer.state_dict()

    save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, filename)

    del optimizer

    hypernetwork.optimizer_state_dict = None  # dereference it after saving, to save memory.

    return hypernetwork, filename

def save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, filename):