stable-diffusion-webui

1
from collections import namedtuple
2

3
import numpy as np
4
from tqdm import trange
5

6
import modules.scripts as scripts
7
import gradio as gr
8

9
from modules import processing, shared, sd_samplers, sd_samplers_common
10

11
import torch
12
import k_diffusion as K
13

14
def find_noise_for_image(p, cond, uncond, cfg_scale, steps):
15
    x = p.init_latent
16

17
    s_in = x.new_ones([x.shape[0]])
18
    if shared.sd_model.parameterization == "v":
19
        dnw = K.external.CompVisVDenoiser(shared.sd_model)
20
        skip = 1
21
    else:
22
        dnw = K.external.CompVisDenoiser(shared.sd_model)
23
        skip = 0
24
    sigmas = dnw.get_sigmas(steps).flip(0)
25

26
    shared.state.sampling_steps = steps
27

28
    for i in trange(1, len(sigmas)):
29
        shared.state.sampling_step += 1
30

31
        x_in = torch.cat([x] * 2)
32
        sigma_in = torch.cat([sigmas[i] * s_in] * 2)
33
        cond_in = torch.cat([uncond, cond])
34

35
        image_conditioning = torch.cat([p.image_conditioning] * 2)
36
        cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]}
37

38
        c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]]
39
        t = dnw.sigma_to_t(sigma_in)
40

41
        eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in)
42
        denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2)
43

44
        denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale
45

46
        d = (x - denoised) / sigmas[i]
47
        dt = sigmas[i] - sigmas[i - 1]
48

49
        x = x + d * dt
50

51
        sd_samplers_common.store_latent(x)
52

53
        # This shouldn't be necessary, but solved some VRAM issues
54
        del x_in, sigma_in, cond_in, c_out, c_in, t,
55
        del eps, denoised_uncond, denoised_cond, denoised, d, dt
56

57
    shared.state.nextjob()
58

59
    return x / x.std()
60

61

62
Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment"])
63

64

65
# Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736
66
def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps):
67
    x = p.init_latent
68

69
    s_in = x.new_ones([x.shape[0]])
70
    if shared.sd_model.parameterization == "v":
71
        dnw = K.external.CompVisVDenoiser(shared.sd_model)
72
        skip = 1
73
    else:
74
        dnw = K.external.CompVisDenoiser(shared.sd_model)
75
        skip = 0
76
    sigmas = dnw.get_sigmas(steps).flip(0)
77

78
    shared.state.sampling_steps = steps
79

80
    for i in trange(1, len(sigmas)):
81
        shared.state.sampling_step += 1
82

83
        x_in = torch.cat([x] * 2)
84
        sigma_in = torch.cat([sigmas[i - 1] * s_in] * 2)
85
        cond_in = torch.cat([uncond, cond])
86

87
        image_conditioning = torch.cat([p.image_conditioning] * 2)
88
        cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]}
89

90
        c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]]
91

92
        if i == 1:
93
            t = dnw.sigma_to_t(torch.cat([sigmas[i] * s_in] * 2))
94
        else:
95
            t = dnw.sigma_to_t(sigma_in)
96

97
        eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in)
98
        denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2)
99

100
        denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale
101

102
        if i == 1:
103
            d = (x - denoised) / (2 * sigmas[i])
104
        else:
105
            d = (x - denoised) / sigmas[i - 1]
106

107
        dt = sigmas[i] - sigmas[i - 1]
108
        x = x + d * dt
109

110
        sd_samplers_common.store_latent(x)
111

112
        # This shouldn't be necessary, but solved some VRAM issues
113
        del x_in, sigma_in, cond_in, c_out, c_in, t,
114
        del eps, denoised_uncond, denoised_cond, denoised, d, dt
115

116
    shared.state.nextjob()
117

118
    return x / sigmas[-1]
119

120

121
class Script(scripts.Script):
122
    def __init__(self):
123
        self.cache = None
124

125
    def title(self):
126
        return "img2img alternative test"
127

128
    def show(self, is_img2img):
129
        return is_img2img
130

131
    def ui(self, is_img2img):
132
        info = gr.Markdown('''
133
        * `CFG Scale` should be 2 or lower.
134
        ''')
135

136
        override_sampler = gr.Checkbox(label="Override `Sampling method` to Euler?(this method is built for it)", value=True, elem_id=self.elem_id("override_sampler"))
137

138
        override_prompt = gr.Checkbox(label="Override `prompt` to the same value as `original prompt`?(and `negative prompt`)", value=True, elem_id=self.elem_id("override_prompt"))
139
        original_prompt = gr.Textbox(label="Original prompt", lines=1, elem_id=self.elem_id("original_prompt"))
140
        original_negative_prompt = gr.Textbox(label="Original negative prompt", lines=1, elem_id=self.elem_id("original_negative_prompt"))
141

142
        override_steps = gr.Checkbox(label="Override `Sampling Steps` to the same value as `Decode steps`?", value=True, elem_id=self.elem_id("override_steps"))
143
        st = gr.Slider(label="Decode steps", minimum=1, maximum=150, step=1, value=50, elem_id=self.elem_id("st"))
144

145
        override_strength = gr.Checkbox(label="Override `Denoising strength` to 1?", value=True, elem_id=self.elem_id("override_strength"))
146

147
        cfg = gr.Slider(label="Decode CFG scale", minimum=0.0, maximum=15.0, step=0.1, value=1.0, elem_id=self.elem_id("cfg"))
148
        randomness = gr.Slider(label="Randomness", minimum=0.0, maximum=1.0, step=0.01, value=0.0, elem_id=self.elem_id("randomness"))
149
        sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=False, elem_id=self.elem_id("sigma_adjustment"))
150

151
        return [
152
            info,
153
            override_sampler,
154
            override_prompt, original_prompt, original_negative_prompt,
155
            override_steps, st,
156
            override_strength,
157
            cfg, randomness, sigma_adjustment,
158
        ]
159

160
    def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment):
161
        # Override
162
        if override_sampler:
163
            p.sampler_name = "Euler"
164
        if override_prompt:
165
            p.prompt = original_prompt
166
            p.negative_prompt = original_negative_prompt
167
        if override_steps:
168
            p.steps = st
169
        if override_strength:
170
            p.denoising_strength = 1.0
171

172
        def sample_extra(conditioning, unconditional_conditioning, seeds, subseeds, subseed_strength, prompts):
173
            lat = (p.init_latent.cpu().numpy() * 10).astype(int)
174

175
            same_params = self.cache is not None and self.cache.cfg_scale == cfg and self.cache.steps == st \
176
                                and self.cache.original_prompt == original_prompt \
177
                                and self.cache.original_negative_prompt == original_negative_prompt \
178
                                and self.cache.sigma_adjustment == sigma_adjustment
179
            same_everything = same_params and self.cache.latent.shape == lat.shape and np.abs(self.cache.latent-lat).sum() < 100
180

181
            if same_everything:
182
                rec_noise = self.cache.noise
183
            else:
184
                shared.state.job_count += 1
185
                cond = p.sd_model.get_learned_conditioning(p.batch_size * [original_prompt])
186
                uncond = p.sd_model.get_learned_conditioning(p.batch_size * [original_negative_prompt])
187
                if sigma_adjustment:
188
                    rec_noise = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st)
189
                else:
190
                    rec_noise = find_noise_for_image(p, cond, uncond, cfg, st)
191
                self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment)
192

193
            rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p)
194

195
            combined_noise = ((1 - randomness) * rec_noise + randomness * rand_noise) / ((randomness**2 + (1-randomness)**2) ** 0.5)
196

197
            sampler = sd_samplers.create_sampler(p.sampler_name, p.sd_model)
198

199
            sigmas = sampler.model_wrap.get_sigmas(p.steps)
200

201
            noise_dt = combined_noise - (p.init_latent / sigmas[0])
202

203
            p.seed = p.seed + 1
204

205
            return sampler.sample_img2img(p, p.init_latent, noise_dt, conditioning, unconditional_conditioning, image_conditioning=p.image_conditioning)
206

207
        p.sample = sample_extra
208

209
        p.extra_generation_params["Decode prompt"] = original_prompt
210
        p.extra_generation_params["Decode negative prompt"] = original_negative_prompt
211
        p.extra_generation_params["Decode CFG scale"] = cfg
212
        p.extra_generation_params["Decode steps"] = st
213
        p.extra_generation_params["Randomness"] = randomness
214
        p.extra_generation_params["Sigma Adjustment"] = sigma_adjustment
215

216
        processed = processing.process_images(p)
217

218
        return processed
219