Merge remote-tracking branch 'origin/test_resolve_conflicts' into test_resolve_conflicts

    args = shlex.split(commandline_args)

    args, skip_torch_cuda_test = extract_arg(args, '--skip-torch-cuda-test')

    args, reinstall_xformers = extract_arg(args, '--reinstall-xformers')

    xformers = '--xformers' in args

    deepdanbooru = '--deepdanbooru' in args

    ngrok = '--ngrok' in args

    if not is_installed("clip"):

        run_pip(f"install {clip_package}", "clip")

    if not is_installed("xformers") and xformers and platform.python_version().startswith("3.10"):

    if (not is_installed("xformers") or reinstall_xformers) and xformers and platform.python_version().startswith("3.10"):

        if platform.system() == "Windows":

            run_pip("install https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/c/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl", "xformers")

            run_pip("install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl", "xformers")

        elif platform.system() == "Linux":

            run_pip("install xformers", "xformers")

            optimizer.zero_grad()

            loss.backward()

            optimizer.step()

        pbar.set_description(f"loss: {losses.mean():.7f}")

        mean_loss = losses.mean()

        if torch.isnan(mean_loss):

            raise RuntimeError("Loss diverged.")

        pbar.set_description(f"loss: {mean_loss:.7f}")

        if hypernetwork.step > 0 and hypernetwork_dir is not None and hypernetwork.step % save_hypernetwork_every == 0:

            last_saved_file = os.path.join(hypernetwork_dir, f'{hypernetwork_name}-{hypernetwork.step}.pt')

            hypernetwork.save(last_saved_file)

        textual_inversion.write_loss(log_directory, "hypernetwork_loss.csv", hypernetwork.step, len(ds), {

            "loss": f"{losses.mean():.7f}",

            "loss": f"{mean_loss:.7f}",

            "learn_rate": scheduler.learn_rate

        })

        shared.state.textinfo = f"""

<p>

Loss: {losses.mean():.7f}<br/>

Loss: {mean_loss:.7f}<br/>

Step: {hypernetwork.step}<br/>

Last prompt: {html.escape(entries[0].cond_text)}<br/>

Last saved embedding: {html.escape(last_saved_file)}<br/>

    ldm.modules.diffusionmodules.model.nonlinearity = silu

    if cmd_opts.force_enable_xformers or (cmd_opts.xformers and shared.xformers_available and torch.version.cuda and (

            6, 0) <= torch.cuda.get_device_capability(shared.device) <= (8, 6)):

    if cmd_opts.force_enable_xformers or (cmd_opts.xformers and shared.xformers_available and torch.version.cuda and (6, 0) <= torch.cuda.get_device_capability(shared.device) <= (9, 0)):

        print("Applying xformers cross attention optimization.")

        ldm.modules.attention.CrossAttention.forward = sd_hijack_optimizations.xformers_attention_forward

        ldm.modules.diffusionmodules.model.AttnBlock.forward = sd_hijack_optimizations.xformers_attnblock_forward

            data = []

            if filename.upper().endswith('.PNG'):

            if os.path.splitext(filename.upper())[-1] in ['.PNG', '.WEBP', '.JXL', '.AVIF']:

                embed_image = Image.open(path)

                if 'sd-ti-embedding' in embed_image.text:

                if hasattr(embed_image, 'text') and 'sd-ti-embedding' in embed_image.text:

                    data = embedding_from_b64(embed_image.text['sd-ti-embedding'])

                    name = data.get('name', name)

                else:

    last_saved_file = "<none>"

    last_saved_image = "<none>"

    embedding_yet_to_be_embedded = False

    ititial_step = embedding.step or 0

    if ititial_step > steps:

        if embedding.step > 0 and embedding_dir is not None and embedding.step % save_embedding_every == 0:

            last_saved_file = os.path.join(embedding_dir, f'{embedding_name}-{embedding.step}.pt')

            embedding.save(last_saved_file)

            embedding_yet_to_be_embedded = True

        write_loss(log_directory, "textual_inversion_loss.csv", embedding.step, len(ds), {

            "loss": f"{losses.mean():.7f}",

            shared.state.current_image = image

            if save_image_with_stored_embedding and os.path.exists(last_saved_file):

            if save_image_with_stored_embedding and os.path.exists(last_saved_file) and embedding_yet_to_be_embedded:

                last_saved_image_chunks = os.path.join(images_embeds_dir, f'{embedding_name}-{embedding.step}.png')

                info.add_text("sd-ti-embedding", embedding_to_b64(data))

                title = "<{}>".format(data.get('name', '???'))

                try:

                    vectorSize = list(data['string_to_param'].values())[0].shape[0]

                except Exception as e:

                    vectorSize = '?'

                checkpoint = sd_models.select_checkpoint()

                footer_left = checkpoint.model_name

                footer_mid = '[{}]'.format(checkpoint.hash)

                footer_right = '{}'.format(embedding.step)

                footer_right = '{}v {}s'.format(vectorSize, embedding.step)

                captioned_image = caption_image_overlay(image, title, footer_left, footer_mid, footer_right)

                captioned_image = insert_image_data_embed(captioned_image, data)

                captioned_image.save(last_saved_image_chunks, "PNG", pnginfo=info)

                embedding_yet_to_be_embedded = False

            image.save(last_saved_image)

            writer.writerow(["prompt", "seed", "width", "height", "sampler", "cfgs", "steps", "filename", "negative_prompt"])

        for image_index, filedata in enumerate(images, start_index):

            if filedata.startswith("data:image/png;base64,"):

                filedata = filedata[len("data:image/png;base64,"):]

            image = Image.open(io.BytesIO(base64.decodebytes(filedata.encode('utf-8'))))

            image = image_from_url_text(filedata)

            is_grid = image_index < p.index_of_first_image

            i = 0 if is_grid else (image_index - p.index_of_first_image)

                    txt2img_preview = gr.Image(elem_id='txt2img_preview', visible=False)

                    txt2img_gallery = gr.Gallery(label='Output', show_label=False, elem_id='txt2img_gallery').style(grid=4)

                with gr.Group():

                with gr.Column():

                    with gr.Row():

                        save = gr.Button('Save')

                        send_to_img2img = gr.Button('Send to img2img')

                    img2img_preview = gr.Image(elem_id='img2img_preview', visible=False)

                    img2img_gallery = gr.Gallery(label='Output', show_label=False, elem_id='img2img_gallery').style(grid=4)

                with gr.Group():

                with gr.Column():

                    with gr.Row():

                        save = gr.Button('Save')

                        img2img_send_to_img2img = gr.Button('Send to img2img')