Revert "fix bugs and optimizations"

            linears.append(torch.nn.Linear(int(dim * layer_structure[i]), int(dim * layer_structure[i+1])))

            # if skip_first_layer because first parameters potentially contain negative values

            # if i < 1: continue

            if add_layer_norm:

                linears.append(torch.nn.LayerNorm(int(dim * layer_structure[i+1])))

            if activation_func in HypernetworkModule.activation_dict:

                linears.append(HypernetworkModule.activation_dict[activation_func]())

            else:

                print("Invalid key {} encountered as activation function!".format(activation_func))

            # if use_dropout: 

            # linears.append(torch.nn.Dropout(p=0.3))

            if add_layer_norm:

                linears.append(torch.nn.LayerNorm(int(dim * layer_structure[i+1])))

        self.linear = torch.nn.Sequential(*linears)

        for k, layers in self.layers.items():

            for layer in layers:

                layer.train()

                res += layer.trainables()

        return res

    def eval(self):

        for k, layers in self.layers.items():

            for layer in layers:

                layer.eval()

        for items in self.weights():

            items.requires_grad = False

    def train(self):

        for k, layers in self.layers.items():

            for layer in layers:

                layer.train()

        for items in self.weights():

            items.requires_grad = True

    def save(self, filename):

        state_dict = {}

        shared.sd_model.first_stage_model.to(devices.cpu)

    hypernetwork = shared.loaded_hypernetwork

    weights = hypernetwork.weights()

    for weight in weights:

        weight.requires_grad = True

    losses = torch.zeros((32,))

    last_saved_file = "<none>"

        return hypernetwork, filename

    scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)

    optimizer = torch.optim.AdamW(hypernetwork.weights(), lr=scheduler.learn_rate)

    # if optimizer == "AdamW": or else Adam / AdamW / SGD, etc...

    optimizer = torch.optim.AdamW(weights, lr=scheduler.learn_rate)

    pbar = tqdm.tqdm(enumerate(ds), total=steps - ititial_step)

    hypernetwork.train()

    for i, entries in pbar:

        hypernetwork.step = i + ititial_step

            losses[hypernetwork.step % losses.shape[0]] = loss.item()

            optimizer.zero_grad(set_to_none=True)

            optimizer.zero_grad()

            loss.backward()

            del loss

            optimizer.step()

        mean_loss = losses.mean()

        if torch.isnan(mean_loss):

        })

        if hypernetwork.step > 0 and images_dir is not None and hypernetwork.step % create_image_every == 0:

            torch.cuda.empty_cache()

            last_saved_image = os.path.join(images_dir, f'{hypernetwork_name}-{hypernetwork.step}.png')

            with torch.no_grad():

                hypernetwork.eval()

            optimizer.zero_grad()

            shared.sd_model.cond_stage_model.to(devices.device)

            shared.sd_model.first_stage_model.to(devices.device)

                image.save(last_saved_image)

                last_saved_image += f", prompt: {preview_text}"

            hypernetwork.train()

        shared.state.job_no = hypernetwork.step

        shared.state.textinfo = f"""