support specifying te and unet weights separately

            params_list.append(extra_networks.ExtraNetworkParams(items=[additional, shared.opts.extra_networks_default_multiplier]))

        names = []

        multipliers = []

        te_multipliers = []

        unet_multipliers = []

        dyn_dims = []

        for params in params_list:

            assert params.items

            names.append(params.items[0])

            multipliers.append(float(params.items[1]) if len(params.items) > 1 else 1.0)

            names.append(params.positional[0])

        networks.load_networks(names, multipliers)

            te_multiplier = float(params.positional[1]) if len(params.positional) > 1 else 1.0

            te_multiplier = float(params.named.get("te", te_multiplier))

            unet_multiplier = float(params.positional[2]) if len(params.positional) > 2 else 1.0

            unet_multiplier = float(params.named.get("unet", unet_multiplier))

            dyn_dim = int(params.positional[3]) if len(params.positional) > 3 else None

            dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else dyn_dim

            te_multipliers.append(te_multiplier)

            unet_multipliers.append(unet_multiplier)

            dyn_dims.append(dyn_dim)

        networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims)

        if shared.opts.lora_add_hashes_to_infotext:

            network_hashes = []

        up = up[:, :dyn_dim]

        down = down[:dyn_dim, :]

    return (up @ down).reshape(shape)

def rebuild_cp_decomposition(up, down, mid):

    up = up.reshape(up.size(0), -1)

    down = down.reshape(down.size(0), -1)

    return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down)

    def __init__(self, name, network_on_disk: NetworkOnDisk):

        self.name = name

        self.network_on_disk = network_on_disk

        self.multiplier = 1.0

        self.te_multiplier = 1.0

        self.unet_multiplier = 1.0

        self.dyn_dim = None

        self.modules = {}

        self.mtime = None

        self.sd_key = weights.sd_key

        self.sd_module = weights.sd_module

        if hasattr(self.sd_module, 'weight'):

            self.shape = self.sd_module.weight.shape

        self.dim = None

        self.bias = weights.w.get("bias")

        self.alpha = weights.w["alpha"].item() if "alpha" in weights.w else None

        self.scale = weights.w["scale"].item() if "scale" in weights.w else None

    def multiplier(self):

        if 'transformer' in self.sd_key[:20]:

            return self.network.te_multiplier

        else:

            return self.network.unet_multiplier

    def calc_scale(self):

        if self.scale is not None:

            return self.scale

        if self.dim is not None and self.alpha is not None:

            return self.alpha / self.dim

        return 1.0

    def finalize_updown(self, updown, orig_weight, output_shape):

        if self.bias is not None:

            updown = updown.reshape(self.bias.shape)

            updown += self.bias.to(orig_weight.device, dtype=orig_weight.dtype)

            updown = updown.reshape(output_shape)

        if len(output_shape) == 4:

            updown = updown.reshape(output_shape)

        if orig_weight.size().numel() == updown.size().numel():

            updown = updown.reshape(orig_weight.shape)

        return updown * self.calc_scale() * self.multiplier()

    def calc_updown(self, target):

        raise NotImplementedError()

import lyco_helpers

import network

class ModuleTypeFull(network.ModuleType):

    def create_module(self, net: network.Network, weights: network.NetworkWeights):

        if all(x in weights.w for x in ["diff"]):

            return NetworkModuleFull(net, weights)

        return None

class NetworkModuleFull(network.NetworkModule):

    def __init__(self,  net: network.Network, weights: network.NetworkWeights):

        super().__init__(net, weights)

        self.weight = weights.w.get("diff")

    def calc_updown(self, orig_weight):

        output_shape = self.weight.shape

        updown = self.weight.to(orig_weight.device, dtype=orig_weight.dtype)

        return self.finalize_updown(updown, orig_weight, output_shape)

import lyco_helpers

import network

import network_lyco

class ModuleTypeHada(network.ModuleType):

        return None

class NetworkModuleHada(network_lyco.NetworkModuleLyco):

class NetworkModuleHada(network.NetworkModule):

    def __init__(self,  net: network.Network, weights: network.NetworkWeights):

        super().__init__(net, weights)