Unverified Commit f1db987e authored by AUTOMATIC1111's avatar AUTOMATIC1111 Committed by GitHub
Browse files

Merge pull request #8958 from MrCheeze/variations-model 

Add support for the unclip (Variations) models, unclip-h and unclip-l
parents e49c4798 1f086003

launch.py

+1 −1
Original line number Diff line number Diff line
@@ -235,7 +235,7 @@ def prepare_environment(): 
    codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')

    blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')



    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "47b6b607fdd31875c9279cd2f4f16b92e4ea958e")

    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")

    taming_transformers_commit_hash = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "24268930bf1dce879235a7fddd0b2355b84d7ea6")

    k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "5b3af030dd83e0297272d861c19477735d0317ec")

    codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")

models/karlo/ViT-L-14_stats.th

0 → 100644
+6.91 KiB

File added.

No diff preview for this file type.

View file

modules/lowvram.py

+6 −4
Original line number Diff line number Diff line
@@ -55,12 +55,12 @@ def setup_for_low_vram(sd_model, use_medvram): 
    if hasattr(sd_model.cond_stage_model, 'model'):

        sd_model.cond_stage_model.transformer = sd_model.cond_stage_model.model



    # remove four big modules, cond, first_stage, depth (if applicable), and unet from the model and then

    # remove several big modules: cond, first_stage, depth/embedder (if applicable), and unet from the model and then

    # send the model to GPU. Then put modules back. the modules will be in CPU.

    stored = sd_model.cond_stage_model.transformer, sd_model.first_stage_model, getattr(sd_model, 'depth_model', None), sd_model.model

    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.model = None, None, None, None

    stored = sd_model.cond_stage_model.transformer, sd_model.first_stage_model, getattr(sd_model, 'depth_model', None), getattr(sd_model, 'embedder', None), sd_model.model

    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.embedder, sd_model.model = None, None, None, None, None

    sd_model.to(devices.device)

    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.model = stored

    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.embedder, sd_model.model = stored



    # register hooks for those the first three models

    sd_model.cond_stage_model.transformer.register_forward_pre_hook(send_me_to_gpu)
@@ -69,6 +69,8 @@ def setup_for_low_vram(sd_model, use_medvram): 
    sd_model.first_stage_model.decode = first_stage_model_decode_wrap

    if sd_model.depth_model:

        sd_model.depth_model.register_forward_pre_hook(send_me_to_gpu)

    if sd_model.embedder:

        sd_model.embedder.register_forward_pre_hook(send_me_to_gpu)

    parents[sd_model.cond_stage_model.transformer] = sd_model.cond_stage_model



    if hasattr(sd_model.cond_stage_model, 'model'):

modules/processing.py

+29 −12
Original line number Diff line number Diff line
@@ -78,11 +78,7 @@ def apply_overlay(image, paste_loc, index, overlays): 




def txt2img_image_conditioning(sd_model, x, width, height):

    if sd_model.model.conditioning_key not in {'hybrid', 'concat'}:

        # Dummy zero conditioning if we're not using inpainting model.

        # Still takes up a bit of memory, but no encoder call.

        # Pretty sure we can just make this a 1x1 image since its not going to be used besides its batch size.

        return x.new_zeros(x.shape[0], 5, 1, 1, dtype=x.dtype, device=x.device)

    if sd_model.model.conditioning_key in {'hybrid', 'concat'}: # Inpainting models



        # The "masked-image" in this case will just be all zeros since the entire image is masked.

        image_conditioning = torch.zeros(x.shape[0], 3, height, width, device=x.device)
@@ -94,6 +90,16 @@ def txt2img_image_conditioning(sd_model, x, width, height): 


        return image_conditioning



    elif sd_model.model.conditioning_key == "crossattn-adm": # UnCLIP models



        return x.new_zeros(x.shape[0], 2*sd_model.noise_augmentor.time_embed.dim, dtype=x.dtype, device=x.device)



    else:

        # Dummy zero conditioning if we're not using inpainting or unclip models.

        # Still takes up a bit of memory, but no encoder call.

        # Pretty sure we can just make this a 1x1 image since its not going to be used besides its batch size.

        return x.new_zeros(x.shape[0], 5, 1, 1, dtype=x.dtype, device=x.device)





class StableDiffusionProcessing:

    """
@@ -190,6 +196,14 @@ class StableDiffusionProcessing: 


        return conditioning_image



    def unclip_image_conditioning(self, source_image):

        c_adm = self.sd_model.embedder(source_image)

        if self.sd_model.noise_augmentor is not None:

            noise_level = 0 # TODO: Allow other noise levels?

            c_adm, noise_level_emb = self.sd_model.noise_augmentor(c_adm, noise_level=repeat(torch.tensor([noise_level]).to(c_adm.device), '1 -> b', b=c_adm.shape[0]))

            c_adm = torch.cat((c_adm, noise_level_emb), 1)

        return c_adm



    def inpainting_image_conditioning(self, source_image, latent_image, image_mask=None):

        self.is_using_inpainting_conditioning = True
@@ -241,6 +255,9 @@ class StableDiffusionProcessing: 
        if self.sampler.conditioning_key in {'hybrid', 'concat'}:

            return self.inpainting_image_conditioning(source_image, latent_image, image_mask=image_mask)



        if self.sampler.conditioning_key == "crossattn-adm":

            return self.unclip_image_conditioning(source_image)



        # Dummy zero conditioning if we're not using inpainting or depth model.

        return latent_image.new_zeros(latent_image.shape[0], 5, 1, 1)

modules/sd_models.py

+8 −0
Original line number Diff line number Diff line
@@ -383,6 +383,14 @@ def repair_config(sd_config): 
    elif shared.cmd_opts.upcast_sampling:

        sd_config.model.params.unet_config.params.use_fp16 = True



    if getattr(sd_config.model.params.first_stage_config.params.ddconfig, "attn_type", None) == "vanilla-xformers" and not shared.xformers_available:

        sd_config.model.params.first_stage_config.params.ddconfig.attn_type = "vanilla"



    # For UnCLIP-L, override the hardcoded karlo directory

    if hasattr(sd_config.model.params, "noise_aug_config") and hasattr(sd_config.model.params.noise_aug_config.params, "clip_stats_path"):

        karlo_path = os.path.join(paths.models_path, 'karlo')

        sd_config.model.params.noise_aug_config.params.clip_stats_path = sd_config.model.params.noise_aug_config.params.clip_stats_path.replace("checkpoints/karlo_models", karlo_path)





sd1_clip_weight = 'cond_stage_model.transformer.text_model.embeddings.token_embedding.weight'

sd2_clip_weight = 'cond_stage_model.model.transformer.resblocks.0.attn.in_proj_weight'

CRA Git | Maintained and supported by SUSTech CRA and CCSE