Merge branch 'master' into DPM++SDE

- API

- Support for dedicated [inpainting model](https://github.com/runwayml/stable-diffusion#inpainting-with-stable-diffusion) by RunwayML. 

- via extension: [Aesthetic Gradients](https://github.com/AUTOMATIC1111/stable-diffusion-webui-aesthetic-gradients), a way to generate images with a specific aesthetic by using clip images embds (implementation of [https://github.com/vicgalle/stable-diffusion-aesthetic-gradients](https://github.com/vicgalle/stable-diffusion-aesthetic-gradients))

## Where are Aesthetic Gradients?!?!

Aesthetic Gradients are now an extension. You can install it using git:

```commandline

git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui-aesthetic-gradients extensions/aesthetic-gradients

```

After running this command, make sure that you have `aesthetic-gradients` dir in webui's `extensions` directory and restart

the UI. The interface for Aesthetic Gradients should appear exactly the same as it was.

## Where is History/Image browser?!?!

Image browser is now an extension. You can install it using git:

```commandline

git clone https://github.com/yfszzx/stable-diffusion-webui-images-browser extensions/images-browser

```

After running this command, make sure that you have `images-browser` dir in webui's `extensions` directory and restart

the UI. The interface for Image browser should appear exactly the same as it was.

- [Stable Diffusion 2.0](https://github.com/Stability-AI/stablediffusion) support - see [wiki](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#stable-diffusion-20) for instructions

## Installation and Running

Make sure the required [dependencies](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Dependencies) are met and follow the instructions available for both [NVidia](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Install-and-Run-on-NVidia-GPUs) (recommended) and [AMD](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Install-and-Run-on-AMD-GPUs) GPUs.

    gfpgan_package = os.environ.get('GFPGAN_PACKAGE', "git+https://github.com/TencentARC/GFPGAN.git@8d2447a2d918f8eba5a4a01463fd48e45126a379")

    clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1")

    openclip_package = os.environ.get('OPENCLIP_PACKAGE', "git+https://github.com/mlfoundations/open_clip.git@bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b")

    xformers_windows_package = os.environ.get('XFORMERS_WINDOWS_PACKAGE', 'https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl')

    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/CompVis/stable-diffusion.git")

    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")

    taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")

    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')

    codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')

    blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')

    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "69ae4b35e0a0f6ee1af8bb9a5d0016ccb27e36dc")

    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "47b6b607fdd31875c9279cd2f4f16b92e4ea958e")

    taming_transformers_commit_hash = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "24268930bf1dce879235a7fddd0b2355b84d7ea6")

    k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "b325595b8a776d483f6935dfa7b45f01c27039e4")

    k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "5b3af030dd83e0297272d861c19477735d0317ec")

    codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")

    blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9")

    if not is_installed("clip"):

        run_pip(f"install {clip_package}", "clip")

    if not is_installed("open_clip"):

        run_pip(f"install {openclip_package}", "open_clip")

    if (not is_installed("xformers") or reinstall_xformers) and xformers:

        if platform.system() == "Windows":

            if platform.python_version().startswith("3.10"):

    os.makedirs(dir_repos, exist_ok=True)

    git_clone(stable_diffusion_repo, repo_dir('stable-diffusion'), "Stable Diffusion", stable_diffusion_commit_hash)

    git_clone(stable_diffusion_repo, repo_dir('stable-diffusion-stability-ai'), "Stable Diffusion", stable_diffusion_commit_hash)

    git_clone(taming_transformers_repo, repo_dir('taming-transformers'), "Taming Transformers", taming_transformers_commit_hash)

    git_clone(k_diffusion_repo, repo_dir('k-diffusion'), "K-diffusion", k_diffusion_commit_hash)

    git_clone(codeformer_repo, repo_dir('CodeFormer'), "CodeFormer", codeformer_commit_hash)

    activation_dict.update({cls_name.lower(): cls_obj for cls_name, cls_obj in inspect.getmembers(torch.nn.modules.activation) if inspect.isclass(cls_obj) and cls_obj.__module__ == 'torch.nn.modules.activation'})

    def __init__(self, dim, state_dict=None, layer_structure=None, activation_func=None, weight_init='Normal',

                 add_layer_norm=False, use_dropout=False, activate_output=False, last_layer_dropout=True):

                 add_layer_norm=False, use_dropout=False, activate_output=False, last_layer_dropout=False):

        super().__init__()

        assert layer_structure is not None, "layer_structure must not be None"

                HypernetworkModule(size, None, self.layer_structure, self.activation_func, self.weight_init,

                                   self.add_layer_norm, self.use_dropout, self.activate_output, last_layer_dropout=self.last_layer_dropout),

            )

        self.eval_mode()

    def weights(self):

        res = []

        for k, layers in self.layers.items():

            for layer in layers:

                res += layer.parameters()

        return res

    def train_mode(self):

        for k, layers in self.layers.items():

            for layer in layers:

                layer.train()

                res += layer.trainables()

                for param in layer.parameters():

                    param.requires_grad = True

        return res

    def eval_mode(self):

        for k, layers in self.layers.items():

            for layer in layers:

                layer.eval()

                for param in layer.parameters():

                    param.requires_grad = False

    def save(self, filename):

        state_dict = {}

def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, data_root, log_directory, training_width, training_height, steps, create_image_every, save_hypernetwork_every, template_file, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):

def train_hypernetwork(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, steps, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_hypernetwork_every, template_file, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):

    # images allows training previews to have infotext. Importing it at the top causes a circular import problem.

    from modules import images

    save_hypernetwork_every = save_hypernetwork_every or 0

    create_image_every = create_image_every or 0

    textual_inversion.validate_train_inputs(hypernetwork_name, learn_rate, batch_size, data_root, template_file, steps, save_hypernetwork_every, create_image_every, log_directory, name="hypernetwork")

    textual_inversion.validate_train_inputs(hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, template_file, steps, save_hypernetwork_every, create_image_every, log_directory, name="hypernetwork")

    path = shared.hypernetworks.get(hypernetwork_name, None)

    shared.loaded_hypernetwork = Hypernetwork()

    hypernetwork = shared.loaded_hypernetwork

    checkpoint = sd_models.select_checkpoint()

    ititial_step = hypernetwork.step or 0

    if ititial_step >= steps:

    initial_step = hypernetwork.step or 0

    if initial_step >= steps:

        shared.state.textinfo = f"Model has already been trained beyond specified max steps"

        return hypernetwork, filename

    scheduler = LearnRateScheduler(learn_rate, steps, ititial_step)

    scheduler = LearnRateScheduler(learn_rate, steps, initial_step)

    # dataset loading may take a while, so input validations and early returns should be done before this

    shared.state.textinfo = f"Preparing dataset from {html.escape(data_root)}..."

    with torch.autocast("cuda"):

        ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size)

    pin_memory = shared.opts.pin_memory

    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method)

    latent_sampling_method = ds.latent_sampling_method

    dl = modules.textual_inversion.dataset.PersonalizedDataLoader(ds, latent_sampling_method=latent_sampling_method, batch_size=ds.batch_size, pin_memory=pin_memory)

    if unload:

        shared.sd_model.cond_stage_model.to(devices.cpu)

        shared.sd_model.first_stage_model.to(devices.cpu)

    size = len(ds.indexes)

    loss_dict = defaultdict(lambda : deque(maxlen = 1024))

    losses = torch.zeros((size,))

    previous_mean_losses = [0]

    previous_mean_loss = 0

    print("Mean loss of {} elements".format(size))

    weights = hypernetwork.weights()

    for weight in weights:

        weight.requires_grad = True

    hypernetwork.train_mode()

    # Here we use optimizer from saved HN, or we can specify as UI option.

    if hypernetwork.optimizer_name in optimizer_dict:

            print("Cannot resume from saved optimizer!")

            print(e)

    scaler = torch.cuda.amp.GradScaler()

    batch_size = ds.batch_size

    gradient_step = ds.gradient_step

    # n steps = batch_size * gradient_step * n image processed

    steps_per_epoch = len(ds) // batch_size // gradient_step

    max_steps_per_epoch = len(ds) // batch_size - (len(ds) // batch_size) % gradient_step

    loss_step = 0

    _loss_step = 0 #internal

    # size = len(ds.indexes)

    # loss_dict = defaultdict(lambda : deque(maxlen = 1024))

    # losses = torch.zeros((size,))

    # previous_mean_losses = [0]

    # previous_mean_loss = 0

    # print("Mean loss of {} elements".format(size))

    steps_without_grad = 0

    last_saved_file = "<none>"

    last_saved_image = "<none>"

    forced_filename = "<none>"

    pbar = tqdm.tqdm(enumerate(ds), total=steps - ititial_step)

    for i, entries in pbar:

        hypernetwork.step = i + ititial_step

        if len(loss_dict) > 0:

            previous_mean_losses = [i[-1] for i in loss_dict.values()]

            previous_mean_loss = mean(previous_mean_losses)

    pbar = tqdm.tqdm(total=steps - initial_step)

    try:

        for i in range((steps-initial_step) * gradient_step):

            if scheduler.finished:

                break

            if shared.state.interrupted:

                break

            for j, batch in enumerate(dl):

                # works as a drop_last=True for gradient accumulation

                if j == max_steps_per_epoch:

                    break

                scheduler.apply(optimizer, hypernetwork.step)

                if scheduler.finished:

                    break

                if shared.state.interrupted:

                    break

                with torch.autocast("cuda"):

            c = stack_conds([entry.cond for entry in entries]).to(devices.device)

            # c = torch.vstack([entry.cond for entry in entries]).to(devices.device)

            x = torch.stack([entry.latent for entry in entries]).to(devices.device)

            loss = shared.sd_model(x, c)[0]

                    x = batch.latent_sample.to(devices.device, non_blocking=pin_memory)

                    if tag_drop_out != 0 or shuffle_tags:

                        shared.sd_model.cond_stage_model.to(devices.device)

                        c = shared.sd_model.cond_stage_model(batch.cond_text).to(devices.device, non_blocking=pin_memory)

                        shared.sd_model.cond_stage_model.to(devices.cpu)

                    else:

                        c = stack_conds(batch.cond).to(devices.device, non_blocking=pin_memory)

                    loss = shared.sd_model(x, c)[0] / gradient_step

                    del x

                    del c

            losses[hypernetwork.step % losses.shape[0]] = loss.item()

            for entry in entries:

                loss_dict[entry.filename].append(loss.item())

            optimizer.zero_grad()

            weights[0].grad = None

            loss.backward()

            if weights[0].grad is None:

                steps_without_grad += 1

            else:

                steps_without_grad = 0

            assert steps_without_grad < 10, 'no gradient found for the trained weight after backward() for 10 steps in a row; this is a bug; training cannot continue'

            optimizer.step()

                    _loss_step += loss.item()

                scaler.scale(loss).backward()

                # go back until we reach gradient accumulation steps

                if (j + 1) % gradient_step != 0:

                    continue

                # print(f"grad:{weights[0].grad.detach().cpu().abs().mean().item():.7f}")

                # scaler.unscale_(optimizer)

                # print(f"grad:{weights[0].grad.detach().cpu().abs().mean().item():.15f}")

                # torch.nn.utils.clip_grad_norm_(weights, max_norm=1.0)

                # print(f"grad:{weights[0].grad.detach().cpu().abs().mean().item():.15f}")

                scaler.step(optimizer)

                scaler.update()

                hypernetwork.step += 1

                pbar.update()

                optimizer.zero_grad(set_to_none=True)

                loss_step = _loss_step

                _loss_step = 0

                steps_done = hypernetwork.step + 1

        if torch.isnan(losses[hypernetwork.step % losses.shape[0]]): 

            raise RuntimeError("Loss diverged.")

        if len(previous_mean_losses) > 1:

            std = stdev(previous_mean_losses)

        else:

            std = 0

        dataset_loss_info = f"dataset loss:{mean(previous_mean_losses):.3f}" + u"\u00B1" + f"({std / (len(previous_mean_losses) ** 0.5):.3f})"

        pbar.set_description(dataset_loss_info)

                epoch_num = hypernetwork.step // steps_per_epoch

                epoch_step = hypernetwork.step % steps_per_epoch

                pbar.set_description(f"[Epoch {epoch_num}: {epoch_step+1}/{steps_per_epoch}]loss: {loss_step:.7f}")

                if hypernetwork_dir is not None and steps_done % save_hypernetwork_every == 0:

                    # Before saving, change name to match current checkpoint.

                    hypernetwork_name_every = f'{hypernetwork_name}-{steps_done}'

                    save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, last_saved_file)

                    hypernetwork.optimizer_state_dict = None  # dereference it after saving, to save memory.

        textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, len(ds), {

            "loss": f"{previous_mean_loss:.7f}",

                textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, steps_per_epoch, {

                    "loss": f"{loss_step:.7f}",

                    "learn_rate": scheduler.learn_rate

                })

                if images_dir is not None and steps_done % create_image_every == 0:

                    forced_filename = f'{hypernetwork_name}-{steps_done}'

                    last_saved_image = os.path.join(images_dir, forced_filename)

            optimizer.zero_grad()

                    hypernetwork.eval_mode()

                    shared.sd_model.cond_stage_model.to(devices.device)

                    shared.sd_model.first_stage_model.to(devices.device)

                        p.width = preview_width

                        p.height = preview_height

                    else:

                p.prompt = entries[0].cond_text

                        p.prompt = batch.cond_text[0]

                        p.steps = 20

                        p.width = training_width

                        p.height = training_height

                    preview_text = p.prompt

                    if unload:

                        shared.sd_model.cond_stage_model.to(devices.cpu)

                        shared.sd_model.first_stage_model.to(devices.cpu)

                    hypernetwork.train_mode()

                    if image is not None:

                        shared.state.current_image = image

                        last_saved_image, last_text_info = images.save_image(image, images_dir, "", p.seed, p.prompt, shared.opts.samples_format, processed.infotexts[0], p=p, forced_filename=forced_filename, save_to_dirs=False)

                shared.state.textinfo = f"""

<p>

Loss: {previous_mean_loss:.7f}<br/>

Step: {hypernetwork.step}<br/>

Last prompt: {html.escape(entries[0].cond_text)}<br/>

Loss: {loss_step:.7f}<br/>

Step: {steps_done}<br/>

Last prompt: {html.escape(batch.cond_text[0])}<br/>

Last saved hypernetwork: {html.escape(last_saved_file)}<br/>

Last saved image: {html.escape(last_saved_image)}<br/>

</p>

"""

    report_statistics(loss_dict)

    except Exception:

        print(traceback.format_exc(), file=sys.stderr)

    finally:

        pbar.leave = False

        pbar.close()

        hypernetwork.eval_mode()

        #report_statistics(loss_dict)

    filename = os.path.join(shared.cmd_opts.hypernetwork_dir, f'{hypernetwork_name}.pt')

    hypernetwork.optimizer_name = optimizer_name

    save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, filename)

    del optimizer

    hypernetwork.optimizer_state_dict = None  # dereference it after saving, to save memory.

    shared.sd_model.cond_stage_model.to(devices.device)

    shared.sd_model.first_stage_model.to(devices.device)

    return hypernetwork, filename

def save_hypernetwork(hypernetwork, checkpoint, hypernetwork_name, filename):

    else:

        image.save(fullfn, quality=opts.jpeg_quality)

    image.already_saved_as = fullfn

    target_side_length = 4000

    oversize = image.width > target_side_length or image.height > target_side_length

    if opts.export_for_4chan and (oversize or os.stat(fullfn).st_size > 4 * 1024 * 1024):

        send_me_to_gpu(first_stage_model, None)

        return first_stage_model_decode(z)

    # for SD1, cond_stage_model is CLIP and its NN is in the tranformer frield, but for SD2, it's open clip, and it's in model field

    if hasattr(sd_model.cond_stage_model, 'model'):

        sd_model.cond_stage_model.transformer = sd_model.cond_stage_model.model

    # remove three big modules, cond, first_stage, and unet from the model and then

    # send the model to GPU. Then put modules back. the modules will be in CPU.

    stored = sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.model

    sd_model.first_stage_model.decode = first_stage_model_decode_wrap

    parents[sd_model.cond_stage_model.transformer] = sd_model.cond_stage_model

    if hasattr(sd_model.cond_stage_model, 'model'):

        sd_model.cond_stage_model.model = sd_model.cond_stage_model.transformer

        del sd_model.cond_stage_model.transformer

    if use_medvram:

        sd_model.model.register_forward_pre_hook(send_me_to_gpu)

    else: