From 65fbefd0337f9deb913c6956a9cfe2155c9c2f5b Mon Sep 17 00:00:00 2001 From: xeonvs Date: Wed, 7 Sep 2022 15:58:25 +0200 Subject: [PATCH 01/10] Added support for launching on Apple Silicon --- modules/esrgan_model.py | 7 +++++-- modules/sd_hijack.py | 5 ++++- modules/shared.py | 9 ++++++--- requirements.txt | 2 ++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/modules/esrgan_model.py b/modules/esrgan_model.py index 3dcef5a6e..2ed1d2739 100644 --- a/modules/esrgan_model.py +++ b/modules/esrgan_model.py @@ -14,8 +14,11 @@ import modules.images def load_model(filename): # this code is adapted from https://github.com/xinntao/ESRGAN - - pretrained_net = torch.load(filename) + if torch.has_mps: + map_l = 'cpu' + else: + map_l = None + pretrained_net = torch.load(filename, map_location=map_l) crt_model = arch.RRDBNet(3, 3, 64, 23, gc=32) if 'conv_first.weight' in pretrained_net: diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 2d26b5f71..9d0637bf7 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -232,7 +232,10 @@ class FrozenCLIPEmbedderWithCustomWords(torch.nn.Module): z = outputs.last_hidden_state # restoring original mean is likely not correct, but it seems to work well to prevent artifacts that happen otherwise - batch_multipliers = torch.asarray(np.array(batch_multipliers)).to(device) + if torch.has_mps: + batch_multipliers = torch.asarray(np.array(batch_multipliers).astype('float32')).to(device) + else: + batch_multipliers = torch.asarray(np.array(batch_multipliers)).to(device) original_mean = z.mean() z *= batch_multipliers.reshape(batch_multipliers.shape + (1,)).expand(z.shape) new_mean = z.mean() diff --git a/modules/shared.py b/modules/shared.py index beb6f9bb0..e529ec27a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -36,9 +36,12 @@ parser.add_argument("--opt-split-attention", action='store_true', help="enable o parser.add_argument("--listen", action='store_true', help="launch gradio with 0.0.0.0 as server name, allowing to respond to network requests") cmd_opts = parser.parse_args() -cpu = torch.device("cpu") -gpu = torch.device("cuda") -device = gpu if torch.cuda.is_available() else cpu +if torch.has_cuda: + device = torch.device("cuda") +elif torch.has_mps: + device = torch.device("mps") +else: + device = torch.device("cpu") batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) parallel_processing_allowed = not cmd_opts.lowvram and not cmd_opts.medvram diff --git a/requirements.txt b/requirements.txt index c9e3f2fce..2eebb029a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,7 @@ omegaconf pytorch_lightning diffusers invisible-watermark +einops +taming-transformers-rom1504 git+https://github.com/crowsonkb/k-diffusion.git git+https://github.com/TencentARC/GFPGAN.git From aaeeef82fa61f719cd5a2590423a99bd1229d5f1 Mon Sep 17 00:00:00 2001 From: xeonvs Date: Wed, 7 Sep 2022 18:09:30 +0200 Subject: [PATCH 02/10] Miss device type for option --medvram --- modules/lowvram.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/lowvram.py b/modules/lowvram.py index 4b78deab7..bd1174915 100644 --- a/modules/lowvram.py +++ b/modules/lowvram.py @@ -2,9 +2,12 @@ import torch module_in_gpu = None cpu = torch.device("cpu") -gpu = torch.device("cuda") -device = gpu if torch.cuda.is_available() else cpu - +if torch.has_cuda: + device = gpu = torch.device("cuda") +elif torch.has_mps: + device = gpu = torch.device("mps") +else: + device = gpu = torch.device("cpu") def setup_for_low_vram(sd_model, use_medvram): parents = {} From b681a8f4f29a849c69bf97b631715469282b5095 Mon Sep 17 00:00:00 2001 From: xeonvs Date: Wed, 7 Sep 2022 18:22:36 +0200 Subject: [PATCH 03/10] rollback requirements --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2eebb029a..c9e3f2fce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,5 @@ omegaconf pytorch_lightning diffusers invisible-watermark -einops -taming-transformers-rom1504 git+https://github.com/crowsonkb/k-diffusion.git git+https://github.com/TencentARC/GFPGAN.git From ba1124b326280202cb583bbdc669fb5303bbd3e3 Mon Sep 17 00:00:00 2001 From: xeonvs Date: Wed, 7 Sep 2022 20:40:32 +0200 Subject: [PATCH 04/10] directly convert list to tensor --- modules/sd_hijack.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 9d0637bf7..1084e2484 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -232,10 +232,7 @@ class FrozenCLIPEmbedderWithCustomWords(torch.nn.Module): z = outputs.last_hidden_state # restoring original mean is likely not correct, but it seems to work well to prevent artifacts that happen otherwise - if torch.has_mps: - batch_multipliers = torch.asarray(np.array(batch_multipliers).astype('float32')).to(device) - else: - batch_multipliers = torch.asarray(np.array(batch_multipliers)).to(device) + batch_multipliers = torch.asarray(batch_multipliers).to(device) original_mean = z.mean() z *= batch_multipliers.reshape(batch_multipliers.shape + (1,)).expand(z.shape) new_mean = z.mean() From 4d5a366f00de906b8ff5b124f7f7b6e771fee155 Mon Sep 17 00:00:00 2001 From: fuzzytent Date: Wed, 7 Sep 2022 21:58:11 +0200 Subject: [PATCH 05/10] Allow copy-pasting images into file inputs --- script.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/script.js b/script.js index 51ace27fc..f2cd8877e 100644 --- a/script.js +++ b/script.js @@ -172,3 +172,19 @@ function submit(){ } return res } + +window.addEventListener('paste', e => { + const files = e.clipboardData.files; + if (!files || files.length !== 1) { + return; + } + if (!['image/png', 'image/gif', 'image/jpeg'].includes(files[0].type)) { + return; + } + [...gradioApp().querySelectorAll('input[type=file][accept="image/x-png,image/gif,image/jpeg"]')] + .filter(input => !input.matches('.\\!hidden input[type=file]')) + .forEach(input => { + input.files = files; + input.dispatchEvent(new Event('change')) + }); +}); From 7045c846435cf5c9547729c60e85c386e78c90ed Mon Sep 17 00:00:00 2001 From: fuzzytent Date: Wed, 7 Sep 2022 22:37:54 +0200 Subject: [PATCH 06/10] Also use alpha channel from img2img input image as mask --- modules/img2img.py | 6 ++++-- modules/ui.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/img2img.py b/modules/img2img.py index 3129798da..1e734ac8b 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -1,5 +1,5 @@ import math -from PIL import Image +from PIL import Image, ImageOps, ImageChops from modules.processing import Processed, StableDiffusionProcessingImg2Img, process_images from modules.shared import opts, state @@ -16,7 +16,9 @@ def img2img(prompt: str, init_img, init_img_with_mask, steps: int, sampler_index if is_inpaint: image = init_img_with_mask['image'] - mask = init_img_with_mask['mask'] + alpha_mask = ImageOps.invert(image.split()[-1]).convert('L').point(lambda x: 255 if x > 0 else 0, mode='1') + mask = ImageChops.lighter(alpha_mask, init_img_with_mask['mask'].convert('L')).convert('RGBA') + image = image.convert('RGB') else: image = init_img mask = None diff --git a/modules/ui.py b/modules/ui.py index f5564d0ef..b1a8c776d 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -323,7 +323,7 @@ def create_ui(txt2img, img2img, run_extras, run_pnginfo): with gr.Group(): switch_mode = gr.Radio(label='Mode', elem_id="img2img_mode", choices=['Redraw whole image', 'Inpaint a part of image', 'Loopback', 'SD upscale'], value='Redraw whole image', type="index", show_label=False) init_img = gr.Image(label="Image for img2img", source="upload", interactive=True, type="pil") - init_img_with_mask = gr.Image(label="Image for inpainting with mask", elem_id="img2maskimg", source="upload", interactive=True, type="pil", tool="sketch", visible=False) + init_img_with_mask = gr.Image(label="Image for inpainting with mask", elem_id="img2maskimg", source="upload", interactive=True, type="pil", tool="sketch", visible=False, image_mode="RGBA") resize_mode = gr.Radio(label="Resize mode", show_label=False, choices=["Just resize", "Crop and resize", "Resize and fill"], type="index", value="Just resize") steps = gr.Slider(minimum=1, maximum=150, step=1, label="Sampling Steps", value=20) From d03e9502b1815e2262eff09e0a0f6b21e18e8609 Mon Sep 17 00:00:00 2001 From: Christian <59421913+Cikmo@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:31:25 +0200 Subject: [PATCH 07/10] Fix not being able to have spaces directory Adds quotes around the PYTHON path so that there can be spaces in parent folders. --- webui.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui.bat b/webui.bat index 055a19b04..0de2ab88d 100644 --- a/webui.bat +++ b/webui.bat @@ -35,7 +35,7 @@ echo Unable to create venv in directory %VENV_DIR% goto :show_stdout_stderr :activate_venv -set PYTHON=%~dp0%VENV_DIR%\Scripts\Python.exe +set PYTHON="%~dp0%VENV_DIR%\Scripts\Python.exe" %PYTHON% --version echo venv %PYTHON% goto :install_torch From 52e071da2a04acfd19cf8f6e69e006bd59937447 Mon Sep 17 00:00:00 2001 From: rewbs Date: Thu, 8 Sep 2022 02:35:26 +0000 Subject: [PATCH 08/10] Add color correction to img2img loopback to avoid a progressive skew to magenta. Based on codedealer's PR to hlky's repo here: https://github.com/sd-webui/stable-diffusion-webui/pull/698/files. --- modules/img2img.py | 28 +++++++++++++++++++++++++++- requirements.txt | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/modules/img2img.py b/modules/img2img.py index 3129798da..2c74842d2 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -1,4 +1,6 @@ import math +import cv2 +import numpy as np from PIL import Image from modules.processing import Processed, StableDiffusionProcessingImg2Img, process_images @@ -57,8 +59,19 @@ def img2img(prompt: str, init_img, init_img_with_mask, steps: int, sampler_index state.job_count = n_iter + do_color_correction = False + try: + from skimage import exposure + do_color_correction = True + except: + print("Install scikit-image to perform color correction on loopback") + + for i in range(n_iter): + if do_color_correction and i == 0: + correction_target = cv2.cvtColor(np.asarray(init_img.copy()), cv2.COLOR_RGB2LAB) + p.n_iter = 1 p.batch_size = 1 p.do_not_save_grid = True @@ -69,8 +82,21 @@ def img2img(prompt: str, init_img, init_img_with_mask, steps: int, sampler_index if initial_seed is None: initial_seed = processed.seed initial_info = processed.info + + init_img = processed.images[0] - p.init_images = [processed.images[0]] + if do_color_correction and correction_target is not None: + print("Colour correcting input...") + init_img = Image.fromarray(cv2.cvtColor(exposure.match_histograms( + cv2.cvtColor( + np.asarray(init_img), + cv2.COLOR_RGB2LAB + ), + correction_target, + channel_axis=2 + ), cv2.COLOR_LAB2RGB).astype("uint8")) + + p.init_images = [init_img] p.seed = processed.seed + 1 p.denoising_strength = max(p.denoising_strength * 0.95, 0.1) history.append(processed.images[0]) diff --git a/requirements.txt b/requirements.txt index c9e3f2fce..ba1bc2815 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,6 @@ omegaconf pytorch_lightning diffusers invisible-watermark +scikit-image git+https://github.com/crowsonkb/k-diffusion.git git+https://github.com/TencentARC/GFPGAN.git From 1e7a36fd79612d924f0aca18061f1b3bd947b02a Mon Sep 17 00:00:00 2001 From: rewbs Date: Thu, 8 Sep 2022 02:53:13 +0000 Subject: [PATCH 09/10] Remove debug print. --- modules/img2img.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/img2img.py b/modules/img2img.py index 2c74842d2..529717859 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -86,7 +86,6 @@ def img2img(prompt: str, init_img, init_img_with_mask, steps: int, sampler_index init_img = processed.images[0] if do_color_correction and correction_target is not None: - print("Colour correcting input...") init_img = Image.fromarray(cv2.cvtColor(exposure.match_histograms( cv2.cvtColor( np.asarray(init_img), From bc12eddb408c3503717b234e1a8bb635049f4a91 Mon Sep 17 00:00:00 2001 From: rewbs Date: Thu, 8 Sep 2022 05:57:22 +0000 Subject: [PATCH 10/10] Add scikit-image dependency to requirements_versions.txt for windows users. --- requirements_versions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_versions.txt b/requirements_versions.txt index 177c6b58e..e8a7470cd 100644 --- a/requirements_versions.txt +++ b/requirements_versions.txt @@ -8,3 +8,4 @@ torch transformers==4.19.2 omegaconf==2.1.1 pytorch_lightning==1.7.2 +scikit-image==0.19.2