From fb58fa62400ba0e2cbe8703796d5573f5f91c398 Mon Sep 17 00:00:00 2001 From: EllangoK Date: Sat, 28 Jan 2023 15:37:01 -0500 Subject: [PATCH 01/72] xyz plot now saves sub grids if opts.grid_save also fixed no draw legend for z grid --- scripts/xyz_grid.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 3df404834..3122f6f66 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -286,23 +286,24 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend print("Unexpected error: draw_xyz_grid failed to return even a single processed image") return Processed(p, []) - grids = [None] * len(zs) + sub_grids = [None] * len(zs) for i in range(len(zs)): start_index = i * len(xs) * len(ys) end_index = start_index + len(xs) * len(ys) grid = images.image_grid(image_cache[start_index:end_index], rows=len(ys)) if draw_legend: grid = images.draw_grid_annotations(grid, cell_size[0], cell_size[1], hor_texts, ver_texts) - - grids[i] = grid + sub_grids[i] = grid if include_sub_grids and len(zs) > 1: processed_result.images.insert(i+1, grid) - original_grid_size = grids[0].size - grids = images.image_grid(grids, rows=1) - processed_result.images[0] = images.draw_grid_annotations(grids, original_grid_size[0], original_grid_size[1], title_texts, [[images.GridAnnotation()]]) + sub_grid_size = sub_grids[0].size + z_grid = images.image_grid(sub_grids, rows=1) + if draw_legend: + z_grid = images.draw_grid_annotations(z_grid, sub_grid_size[0], sub_grid_size[1], title_texts, [[images.GridAnnotation()]]) + processed_result.images[0] = z_grid - return processed_result + return processed_result, sub_grids class SharedSettingsStackHelper(object): @@ -576,7 +577,7 @@ class Script(scripts.Script): return res with SharedSettingsStackHelper(): - processed = draw_xyz_grid( + processed, sub_grids = draw_xyz_grid( p, xs=xs, ys=ys, @@ -592,6 +593,10 @@ class Script(scripts.Script): second_axes_processed=second_axes_processed ) + if opts.grid_save and len(sub_grids) > 1: + for sub_grid in sub_grids: + images.save_image(sub_grid, p.outpath_grids, "xyz_grid", info=grid_infotext[0], extension=opts.grid_format, prompt=p.prompt, seed=processed.seed, grid=True, p=p) + if opts.grid_save: images.save_image(processed.images[0], p.outpath_grids, "xyz_grid", info=grid_infotext[0], extension=opts.grid_format, prompt=p.prompt, seed=processed.seed, grid=True, p=p) From 920fe8057cb325e9835f70c0389499c51cbdd3b5 Mon Sep 17 00:00:00 2001 From: EllangoK Date: Sun, 29 Jan 2023 03:36:16 -0500 Subject: [PATCH 02/72] fixes #7284 btn unbound error --- modules/ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ui.py b/modules/ui.py index f11956920..7e193240c 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -466,8 +466,8 @@ def create_ui(): width = gr.Slider(minimum=64, maximum=2048, step=8, label="Width", value=512, elem_id="txt2img_width") height = gr.Slider(minimum=64, maximum=2048, step=8, label="Height", value=512, elem_id="txt2img_height") + res_switch_btn = ToolButton(value=switch_values_symbol, elem_id="txt2img_res_switch_btn") if opts.dimensions_and_batch_together: - res_switch_btn = ToolButton(value=switch_values_symbol, elem_id="txt2img_res_switch_btn") with gr.Column(elem_id="txt2img_column_batch"): batch_count = gr.Slider(minimum=1, step=1, label='Batch count', value=1, elem_id="txt2img_batch_count") batch_size = gr.Slider(minimum=1, maximum=8, step=1, label='Batch size', value=1, elem_id="txt2img_batch_size") @@ -737,8 +737,8 @@ def create_ui(): width = gr.Slider(minimum=64, maximum=2048, step=8, label="Width", value=512, elem_id="img2img_width") height = gr.Slider(minimum=64, maximum=2048, step=8, label="Height", value=512, elem_id="img2img_height") + res_switch_btn = ToolButton(value=switch_values_symbol, elem_id="img2img_res_switch_btn") if opts.dimensions_and_batch_together: - res_switch_btn = ToolButton(value=switch_values_symbol, elem_id="img2img_res_switch_btn") with gr.Column(elem_id="img2img_column_batch"): batch_count = gr.Slider(minimum=1, step=1, label='Batch count', value=1, elem_id="img2img_batch_count") batch_size = gr.Slider(minimum=1, maximum=8, step=1, label='Batch size', value=1, elem_id="img2img_batch_size") From 7c53f81caf817a7e7dc9c2fafebfcce269ecb1d7 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 29 Jan 2023 15:29:03 +0200 Subject: [PATCH 03/72] Prompt selector for Prompt Matrix script --- scripts/prompt_matrix.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/prompt_matrix.py b/scripts/prompt_matrix.py index dd95e5887..702870cef 100644 --- a/scripts/prompt_matrix.py +++ b/scripts/prompt_matrix.py @@ -10,7 +10,7 @@ from modules import images from modules.processing import process_images, Processed from modules.shared import opts, cmd_opts, state import modules.sd_samplers - +from pprint import pprint def draw_xy_grid(xs, ys, x_label, y_label, cell): res = [] @@ -44,16 +44,23 @@ class Script(scripts.Script): def title(self): return "Prompt matrix" - def ui(self, is_img2img): + def ui(self, is_img2img): put_at_start = gr.Checkbox(label='Put variable parts at start of prompt', value=False, elem_id=self.elem_id("put_at_start")) different_seeds = gr.Checkbox(label='Use different seed for each picture', value=False, elem_id=self.elem_id("different_seeds")) + # Radio buttons for selecting the prompt between positive and negative + prompt_type = gr.Radio(["positive", "negative"], label="Select prompt", elem_id=self.elem_id("prompt_type"), value="positive") - return [put_at_start, different_seeds] + return [put_at_start, different_seeds, prompt_type] - def run(self, p, put_at_start, different_seeds): + def run(self, p, put_at_start, different_seeds, prompt_type): modules.processing.fix_seed(p) + # Raise error if promp type is not positive or negative + if prompt_type not in ["positive", "negative"]: + raise ValueError(f"Unknown prompt type {prompt_type}") - original_prompt = p.prompt[0] if type(p.prompt) == list else p.prompt + prompt = p.prompt if prompt_type == "positive" else p.negative_prompt + original_prompt = prompt[0] if type(prompt) == list else prompt + positive_prompt = p.prompt[0] if type(p.prompt) == list else p.prompt all_prompts = [] prompt_matrix_parts = original_prompt.split("|") @@ -73,9 +80,12 @@ class Script(scripts.Script): print(f"Prompt matrix will create {len(all_prompts)} images using a total of {p.n_iter} batches.") - p.prompt = all_prompts + if prompt_type == "positive": + p.prompt = all_prompts + else: + p.negative_prompt = all_prompts p.seed = [p.seed + (i if different_seeds else 0) for i in range(len(all_prompts))] - p.prompt_for_display = original_prompt + p.prompt_for_display = positive_prompt processed = process_images(p) grid = images.image_grid(processed.images, p.batch_size, rows=1 << ((len(prompt_matrix_parts) - 1) // 2)) From edabd927296d389856ffc87c7789111a21a051a2 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 29 Jan 2023 16:05:59 +0200 Subject: [PATCH 04/72] Add delimiter selector to the Prompt Matrix script --- scripts/prompt_matrix.py | 47 ++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/scripts/prompt_matrix.py b/scripts/prompt_matrix.py index 702870cef..89db9e63b 100644 --- a/scripts/prompt_matrix.py +++ b/scripts/prompt_matrix.py @@ -12,6 +12,7 @@ from modules.shared import opts, cmd_opts, state import modules.sd_samplers from pprint import pprint + def draw_xy_grid(xs, ys, x_label, y_label, cell): res = [] @@ -33,7 +34,8 @@ def draw_xy_grid(xs, ys, x_label, y_label, cell): res.append(processed.images[0]) grid = images.image_grid(res, rows=len(ys)) - grid = images.draw_grid_annotations(grid, res[0].width, res[0].height, hor_texts, ver_texts) + grid = images.draw_grid_annotations( + grid, res[0].width, res[0].height, hor_texts, ver_texts) first_processed.images = [grid] @@ -45,56 +47,73 @@ class Script(scripts.Script): return "Prompt matrix" def ui(self, is_img2img): - put_at_start = gr.Checkbox(label='Put variable parts at start of prompt', value=False, elem_id=self.elem_id("put_at_start")) - different_seeds = gr.Checkbox(label='Use different seed for each picture', value=False, elem_id=self.elem_id("different_seeds")) + put_at_start = gr.Checkbox(label='Put variable parts at start of prompt', + value=False, elem_id=self.elem_id("put_at_start")) + different_seeds = gr.Checkbox( + label='Use different seed for each picture', value=False, elem_id=self.elem_id("different_seeds")) # Radio buttons for selecting the prompt between positive and negative - prompt_type = gr.Radio(["positive", "negative"], label="Select prompt", elem_id=self.elem_id("prompt_type"), value="positive") + prompt_type = gr.Radio(["positive", "negative"], label="Select prompt", + elem_id=self.elem_id("prompt_type"), value="positive") + # Radio buttons for selecting the delimiter to use in the resulting prompt + variations_delimiter = gr.Radio(["comma", "space"], label="Select delimiter", elem_id=self.elem_id( + "variations_delimiter"), value="comma") + return [put_at_start, different_seeds, prompt_type, variations_delimiter] - return [put_at_start, different_seeds, prompt_type] - - def run(self, p, put_at_start, different_seeds, prompt_type): + def run(self, p, put_at_start, different_seeds, prompt_type, variations_delimiter): modules.processing.fix_seed(p) # Raise error if promp type is not positive or negative if prompt_type not in ["positive", "negative"]: raise ValueError(f"Unknown prompt type {prompt_type}") + # Raise error if variations delimiter is not comma or space + if variations_delimiter not in ["comma", "space"]: + raise ValueError( + f"Unknown variations delimiter {variations_delimiter}") prompt = p.prompt if prompt_type == "positive" else p.negative_prompt original_prompt = prompt[0] if type(prompt) == list else prompt positive_prompt = p.prompt[0] if type(p.prompt) == list else p.prompt + delimiter = ", " if variations_delimiter == "comma" else " " + all_prompts = [] prompt_matrix_parts = original_prompt.split("|") combination_count = 2 ** (len(prompt_matrix_parts) - 1) for combination_num in range(combination_count): - selected_prompts = [text.strip().strip(',') for n, text in enumerate(prompt_matrix_parts[1:]) if combination_num & (1 << n)] + selected_prompts = [text.strip().strip(',') for n, text in enumerate( + prompt_matrix_parts[1:]) if combination_num & (1 << n)] if put_at_start: selected_prompts = selected_prompts + [prompt_matrix_parts[0]] else: selected_prompts = [prompt_matrix_parts[0]] + selected_prompts - all_prompts.append(", ".join(selected_prompts)) + all_prompts.append(delimiter.join(selected_prompts)) p.n_iter = math.ceil(len(all_prompts) / p.batch_size) p.do_not_save_grid = True - print(f"Prompt matrix will create {len(all_prompts)} images using a total of {p.n_iter} batches.") + print( + f"Prompt matrix will create {len(all_prompts)} images using a total of {p.n_iter} batches.") if prompt_type == "positive": p.prompt = all_prompts else: p.negative_prompt = all_prompts - p.seed = [p.seed + (i if different_seeds else 0) for i in range(len(all_prompts))] + p.seed = [p.seed + (i if different_seeds else 0) + for i in range(len(all_prompts))] p.prompt_for_display = positive_prompt processed = process_images(p) - grid = images.image_grid(processed.images, p.batch_size, rows=1 << ((len(prompt_matrix_parts) - 1) // 2)) - grid = images.draw_prompt_matrix(grid, p.width, p.height, prompt_matrix_parts) + grid = images.image_grid(processed.images, p.batch_size, rows=1 << ( + (len(prompt_matrix_parts) - 1) // 2)) + grid = images.draw_prompt_matrix( + grid, p.width, p.height, prompt_matrix_parts) processed.images.insert(0, grid) processed.index_of_first_image = 1 processed.infotexts.insert(0, processed.infotexts[0]) if opts.grid_save: - images.save_image(processed.images[0], p.outpath_grids, "prompt_matrix", extension=opts.grid_format, prompt=original_prompt, seed=processed.seed, grid=True, p=p) + images.save_image(processed.images[0], p.outpath_grids, "prompt_matrix", + extension=opts.grid_format, prompt=original_prompt, seed=processed.seed, grid=True, p=p) return processed From 5997457fd48c9c0fc31c9d96bf0b9c217585c526 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 29 Jan 2023 16:23:29 +0200 Subject: [PATCH 05/72] Compact options UI for Prompt Matrix --- scripts/prompt_matrix.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/prompt_matrix.py b/scripts/prompt_matrix.py index 89db9e63b..03212b31a 100644 --- a/scripts/prompt_matrix.py +++ b/scripts/prompt_matrix.py @@ -47,16 +47,23 @@ class Script(scripts.Script): return "Prompt matrix" def ui(self, is_img2img): - put_at_start = gr.Checkbox(label='Put variable parts at start of prompt', - value=False, elem_id=self.elem_id("put_at_start")) - different_seeds = gr.Checkbox( - label='Use different seed for each picture', value=False, elem_id=self.elem_id("different_seeds")) - # Radio buttons for selecting the prompt between positive and negative - prompt_type = gr.Radio(["positive", "negative"], label="Select prompt", - elem_id=self.elem_id("prompt_type"), value="positive") - # Radio buttons for selecting the delimiter to use in the resulting prompt - variations_delimiter = gr.Radio(["comma", "space"], label="Select delimiter", elem_id=self.elem_id( - "variations_delimiter"), value="comma") + gr.HTML('
') + with gr.Row(): + with gr.Column(): + put_at_start = gr.Checkbox(label='Put variable parts at start of prompt', + value=False, elem_id=self.elem_id("put_at_start")) + with gr.Column(): + # Radio buttons for selecting the prompt between positive and negative + prompt_type = gr.Radio(["positive", "negative"], label="Select prompt", + elem_id=self.elem_id("prompt_type"), value="positive") + with gr.Row(): + with gr.Column(): + different_seeds = gr.Checkbox( + label='Use different seed for each picture', value=False, elem_id=self.elem_id("different_seeds")) + with gr.Column(): + # Radio buttons for selecting the delimiter to use in the resulting prompt + variations_delimiter = gr.Radio(["comma", "space"], label="Select delimiter", elem_id=self.elem_id( + "variations_delimiter"), value="comma") return [put_at_start, different_seeds, prompt_type, variations_delimiter] def run(self, p, put_at_start, different_seeds, prompt_type, variations_delimiter): From 1e2b10d2dcdf41a6cce0c525c85ebd42a521e0f1 Mon Sep 17 00:00:00 2001 From: Yevhenii Hurin Date: Sun, 29 Jan 2023 17:14:46 +0200 Subject: [PATCH 06/72] Cleanup changes made by formatter --- scripts/prompt_matrix.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/scripts/prompt_matrix.py b/scripts/prompt_matrix.py index 03212b31a..de921ea84 100644 --- a/scripts/prompt_matrix.py +++ b/scripts/prompt_matrix.py @@ -10,7 +10,6 @@ from modules import images from modules.processing import process_images, Processed from modules.shared import opts, cmd_opts, state import modules.sd_samplers -from pprint import pprint def draw_xy_grid(xs, ys, x_label, y_label, cell): @@ -34,8 +33,7 @@ def draw_xy_grid(xs, ys, x_label, y_label, cell): res.append(processed.images[0]) grid = images.image_grid(res, rows=len(ys)) - grid = images.draw_grid_annotations( - grid, res[0].width, res[0].height, hor_texts, ver_texts) + grid = images.draw_grid_annotations(grid, res[0].width, res[0].height, hor_texts, ver_texts) first_processed.images = [grid] @@ -73,8 +71,7 @@ class Script(scripts.Script): raise ValueError(f"Unknown prompt type {prompt_type}") # Raise error if variations delimiter is not comma or space if variations_delimiter not in ["comma", "space"]: - raise ValueError( - f"Unknown variations delimiter {variations_delimiter}") + raise ValueError(f"Unknown variations delimiter {variations_delimiter}") prompt = p.prompt if prompt_type == "positive" else p.negative_prompt original_prompt = prompt[0] if type(prompt) == list else prompt @@ -86,8 +83,7 @@ class Script(scripts.Script): prompt_matrix_parts = original_prompt.split("|") combination_count = 2 ** (len(prompt_matrix_parts) - 1) for combination_num in range(combination_count): - selected_prompts = [text.strip().strip(',') for n, text in enumerate( - prompt_matrix_parts[1:]) if combination_num & (1 << n)] + selected_prompts = [text.strip().strip(',') for n, text in enumerate(prompt_matrix_parts[1:]) if combination_num & (1 << n)] if put_at_start: selected_prompts = selected_prompts + [prompt_matrix_parts[0]] @@ -99,28 +95,23 @@ class Script(scripts.Script): p.n_iter = math.ceil(len(all_prompts) / p.batch_size) p.do_not_save_grid = True - print( - f"Prompt matrix will create {len(all_prompts)} images using a total of {p.n_iter} batches.") + print(f"Prompt matrix will create {len(all_prompts)} images using a total of {p.n_iter} batches.") if prompt_type == "positive": p.prompt = all_prompts else: p.negative_prompt = all_prompts - p.seed = [p.seed + (i if different_seeds else 0) - for i in range(len(all_prompts))] + p.seed = [p.seed + (i if different_seeds else 0) for i in range(len(all_prompts))] p.prompt_for_display = positive_prompt processed = process_images(p) - grid = images.image_grid(processed.images, p.batch_size, rows=1 << ( - (len(prompt_matrix_parts) - 1) // 2)) - grid = images.draw_prompt_matrix( - grid, p.width, p.height, prompt_matrix_parts) + grid = images.image_grid(processed.images, p.batch_size, rows=1 << ((len(prompt_matrix_parts) - 1) // 2)) + grid = images.draw_prompt_matrix(grid, p.width, p.height, prompt_matrix_parts) processed.images.insert(0, grid) processed.index_of_first_image = 1 processed.infotexts.insert(0, processed.infotexts[0]) if opts.grid_save: - images.save_image(processed.images[0], p.outpath_grids, "prompt_matrix", - extension=opts.grid_format, prompt=original_prompt, seed=processed.seed, grid=True, p=p) + images.save_image(processed.images[0], p.outpath_grids, "prompt_matrix", extension=opts.grid_format, prompt=original_prompt, seed=processed.seed, grid=True, p=p) return processed From bfe7e7f15fbceccd016957769cd5b5a26c82a45b Mon Sep 17 00:00:00 2001 From: Piotr Pomierski Date: Tue, 31 Jan 2023 01:51:07 +0100 Subject: [PATCH 07/72] Fix missing tooltip for 'Clear prompt' button --- javascript/hints.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript/hints.js b/javascript/hints.js index 7b60b25e8..75792d0dd 100644 --- a/javascript/hints.js +++ b/javascript/hints.js @@ -17,7 +17,7 @@ titles = { "\u2199\ufe0f": "Read generation parameters from prompt or last generation if prompt is empty into user interface.", "\u{1f4c2}": "Open images output directory", "\u{1f4be}": "Save style", - "\U0001F5D1": "Clear prompt", + "\u{1f5d1}": "Clear prompt", "\u{1f4cb}": "Apply selected styles to current prompt", "\u{1f4d2}": "Paste available values into the field", "\u{1f3b4}": "Show extra networks", From 0426b3478937e54446337cf435ed3f548688b120 Mon Sep 17 00:00:00 2001 From: Joey Sanchez Date: Mon, 30 Jan 2023 21:46:13 -0500 Subject: [PATCH 08/72] Adding default true to use_original_name_batch as images should by default hold the same name to help keep sequenced images in their correct order --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 69634fd80..5600d480c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -327,7 +327,7 @@ options_templates.update(options_section(('saving-images', "Saving images/grids" "jpeg_quality": OptionInfo(80, "Quality for saved jpeg images", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "export_for_4chan": OptionInfo(True, "If PNG image is larger than 4MB or any dimension is larger than 4000, downscale and save copy as JPG"), - "use_original_name_batch": OptionInfo(False, "Use original name for output filename during batch process in extras tab"), + "use_original_name_batch": OptionInfo(True, "Use original name for output filename during batch process in extras tab"), "use_upscaler_name_as_suffix": OptionInfo(False, "Use upscaler name as filename suffix in the extras tab"), "save_selected_only": OptionInfo(True, "When using 'Save' button, only save a single selected image"), "do_not_add_watermark": OptionInfo(False, "Do not add watermark to images"), From 7738c057ce938ca5c5a53a95e2023d3bcf14f06a Mon Sep 17 00:00:00 2001 From: brkirch Date: Wed, 1 Feb 2023 05:23:58 -0500 Subject: [PATCH 09/72] MPS fix is still needed :( Apparently I did not test with large enough images to trigger the bug with torch.narrow on MPS --- modules/devices.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/devices.py b/modules/devices.py index 655ca1d3f..f4afb8975 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -207,3 +207,6 @@ if has_mps(): cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0)) torch.cumsum = lambda input, *args, **kwargs: ( cumsum_fix(input, orig_cumsum, *args, **kwargs) ) torch.Tensor.cumsum = lambda self, *args, **kwargs: ( cumsum_fix(self, orig_Tensor_cumsum, *args, **kwargs) ) + orig_narrow = torch.narrow + torch.narrow = lambda *args, **kwargs: ( orig_narrow(*args, **kwargs).clone() ) + From 2217331cd1245d0bdda786a5dcaf4f7b843bd7e4 Mon Sep 17 00:00:00 2001 From: brkirch Date: Wed, 1 Feb 2023 06:20:19 -0500 Subject: [PATCH 10/72] Refactor MPS fixes to CondFunc --- modules/devices.py | 50 +++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index f4afb8975..919048d0d 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -2,6 +2,7 @@ import sys, os, shlex import contextlib import torch from modules import errors +from modules.sd_hijack_utils import CondFunc from packaging import version @@ -156,36 +157,7 @@ def test_for_nans(x, where): raise NansException(message) -# MPS workaround for https://github.com/pytorch/pytorch/issues/79383 -orig_tensor_to = torch.Tensor.to -def tensor_to_fix(self, *args, **kwargs): - if self.device.type != 'mps' and \ - ((len(args) > 0 and isinstance(args[0], torch.device) and args[0].type == 'mps') or \ - (isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')): - self = self.contiguous() - return orig_tensor_to(self, *args, **kwargs) - - -# MPS workaround for https://github.com/pytorch/pytorch/issues/80800 -orig_layer_norm = torch.nn.functional.layer_norm -def layer_norm_fix(*args, **kwargs): - if len(args) > 0 and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps': - args = list(args) - args[0] = args[0].contiguous() - return orig_layer_norm(*args, **kwargs) - - -# MPS workaround for https://github.com/pytorch/pytorch/issues/90532 -orig_tensor_numpy = torch.Tensor.numpy -def numpy_fix(self, *args, **kwargs): - if self.requires_grad: - self = self.detach() - return orig_tensor_numpy(self, *args, **kwargs) - - # MPS workaround for https://github.com/pytorch/pytorch/issues/89784 -orig_cumsum = torch.cumsum -orig_Tensor_cumsum = torch.Tensor.cumsum def cumsum_fix(input, cumsum_func, *args, **kwargs): if input.device.type == 'mps': output_dtype = kwargs.get('dtype', input.dtype) @@ -199,14 +171,20 @@ def cumsum_fix(input, cumsum_func, *args, **kwargs): if has_mps(): if version.parse(torch.__version__) < version.parse("1.13"): # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working - torch.Tensor.to = tensor_to_fix - torch.nn.functional.layer_norm = layer_norm_fix - torch.Tensor.numpy = numpy_fix + + # MPS workaround for https://github.com/pytorch/pytorch/issues/79383 + CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs), + lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')) + # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 + CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs), + lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps') + # MPS workaround for https://github.com/pytorch/pytorch/issues/90532 + CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad) elif version.parse(torch.__version__) > version.parse("1.13.1"): cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0)) cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0)) - torch.cumsum = lambda input, *args, **kwargs: ( cumsum_fix(input, orig_cumsum, *args, **kwargs) ) - torch.Tensor.cumsum = lambda self, *args, **kwargs: ( cumsum_fix(self, orig_Tensor_cumsum, *args, **kwargs) ) - orig_narrow = torch.narrow - torch.narrow = lambda *args, **kwargs: ( orig_narrow(*args, **kwargs).clone() ) + cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs) + CondFunc('torch.cumsum', cumsum_fix_func, None) + CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None) + CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None) From 1b8af15f13cba2bfce249d9837660ea4f28d451e Mon Sep 17 00:00:00 2001 From: brkirch Date: Wed, 1 Feb 2023 09:28:16 -0500 Subject: [PATCH 11/72] Refactor Mac specific code to a separate file Move most Mac related code to a separate file, don't even load it unless web UI is run under macOS. --- modules/devices.py | 52 +++++--------------------------- modules/mac_specific.py | 56 +++++++++++++++++++++++++++++++++++ modules/sd_samplers_common.py | 16 ---------- modules/shared.py | 3 ++ 4 files changed, 66 insertions(+), 61 deletions(-) create mode 100644 modules/mac_specific.py diff --git a/modules/devices.py b/modules/devices.py index 919048d0d..52c3e7cd7 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -1,22 +1,17 @@ -import sys, os, shlex +import sys import contextlib import torch from modules import errors -from modules.sd_hijack_utils import CondFunc -from packaging import version + +if sys.platform == "darwin": + from modules import mac_specific -# has_mps is only available in nightly pytorch (for now) and macOS 12.3+. -# check `getattr` and try it for compatibility def has_mps() -> bool: - if not getattr(torch, 'has_mps', False): + if sys.platform != "darwin": return False - try: - torch.zeros(1).to(torch.device("mps")) - return True - except Exception: - return False - + else: + return mac_specific.has_mps def extract_device_id(args, name): for x in range(len(args)): @@ -155,36 +150,3 @@ def test_for_nans(x, where): message += " Use --disable-nan-check commandline argument to disable this check." raise NansException(message) - - -# MPS workaround for https://github.com/pytorch/pytorch/issues/89784 -def cumsum_fix(input, cumsum_func, *args, **kwargs): - if input.device.type == 'mps': - output_dtype = kwargs.get('dtype', input.dtype) - if output_dtype == torch.int64: - return cumsum_func(input.cpu(), *args, **kwargs).to(input.device) - elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16): - return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64) - return cumsum_func(input, *args, **kwargs) - - -if has_mps(): - if version.parse(torch.__version__) < version.parse("1.13"): - # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working - - # MPS workaround for https://github.com/pytorch/pytorch/issues/79383 - CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs), - lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')) - # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 - CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs), - lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps') - # MPS workaround for https://github.com/pytorch/pytorch/issues/90532 - CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad) - elif version.parse(torch.__version__) > version.parse("1.13.1"): - cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0)) - cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0)) - cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs) - CondFunc('torch.cumsum', cumsum_fix_func, None) - CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None) - CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None) - diff --git a/modules/mac_specific.py b/modules/mac_specific.py new file mode 100644 index 000000000..e39d670eb --- /dev/null +++ b/modules/mac_specific.py @@ -0,0 +1,56 @@ +import torch +from modules import paths +from modules.sd_hijack_utils import CondFunc +from packaging import version + + +device = None + + +# has_mps is only available in nightly pytorch (for now) and macOS 12.3+. +# check `getattr` and try it for compatibility +def check_for_mps() -> bool: + if not getattr(torch, 'has_mps', False): + return False + try: + torch.zeros(1).to(torch.device("mps")) + return True + except Exception: + return False +has_mps = check_for_mps() + + +# MPS workaround for https://github.com/pytorch/pytorch/issues/89784 +def cumsum_fix(input, cumsum_func, *args, **kwargs): + if input.device.type == 'mps': + output_dtype = kwargs.get('dtype', input.dtype) + if output_dtype == torch.int64: + return cumsum_func(input.cpu(), *args, **kwargs).to(input.device) + elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16): + return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64) + return cumsum_func(input, *args, **kwargs) + + +if has_mps: + # MPS fix for randn in torchsde + CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'mps') + + if version.parse(torch.__version__) < version.parse("1.13"): + # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working + + # MPS workaround for https://github.com/pytorch/pytorch/issues/79383 + CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs), + lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')) + # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 + CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs), + lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps') + # MPS workaround for https://github.com/pytorch/pytorch/issues/90532 + CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad) + elif version.parse(torch.__version__) > version.parse("1.13.1"): + cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0)) + cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0)) + cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs) + CondFunc('torch.cumsum', cumsum_fix_func, None) + CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None) + CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None) + diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 3c03d442e..a1aac7cf0 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -2,7 +2,6 @@ from collections import namedtuple import numpy as np import torch from PIL import Image -import torchsde._brownian.brownian_interval from modules import devices, processing, images, sd_vae_approx from modules.shared import opts, state @@ -61,18 +60,3 @@ def store_latent(decoded): class InterruptedException(BaseException): pass - - -# MPS fix for randn in torchsde -# XXX move this to separate file for MPS -def torchsde_randn(size, dtype, device, seed): - if device.type == 'mps': - generator = torch.Generator(devices.cpu).manual_seed(int(seed)) - return torch.randn(size, dtype=dtype, device=devices.cpu, generator=generator).to(device) - else: - generator = torch.Generator(device).manual_seed(int(seed)) - return torch.randn(size, dtype=dtype, device=device, generator=generator) - - -torchsde._brownian.brownian_interval._randn = torchsde_randn - diff --git a/modules/shared.py b/modules/shared.py index 5600d480c..59f12cd8d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -145,6 +145,9 @@ devices.device, devices.device_interrogate, devices.device_gfpgan, devices.devic (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer']) device = devices.device +if sys.platform == "darwin": + from modules import mac_specific + mac_specific.device = device weight_load_location = None if cmd_opts.lowram else "cpu" batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) From 92bae77b88fd90743eebec69ca7af1ee1c6e40f2 Mon Sep 17 00:00:00 2001 From: ctwrs <> Date: Wed, 1 Feb 2023 21:58:09 +0100 Subject: [PATCH 12/72] Add .jpg to allowed thumb formats --- modules/ui_extra_networks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 833679680..95b30f4ab 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -29,8 +29,9 @@ def add_pages_to_demo(app): if not any([Path(x).resolve() in Path(filename).resolve().parents for x in allowed_dirs]): raise ValueError(f"File cannot be fetched: {filename}. Must be in one of directories registered by extra pages.") - if os.path.splitext(filename)[1].lower() != ".png": - raise ValueError(f"File cannot be fetched: {filename}. Only png.") + ext = os.path.splitext(filename)[1].lower() + if ext not in (".png", ".jpg"): + raise ValueError(f"File cannot be fetched: {filename}. Only png and jpg.") # would profit from returning 304 return FileResponse(filename, headers={"Accept-Ranges": "bytes"}) From fb97acef63ef50d1612566e47c5c0ba4823bd29f Mon Sep 17 00:00:00 2001 From: Cody Brownstein Date: Wed, 1 Feb 2023 14:46:13 -0800 Subject: [PATCH 13/72] Update error message WRT missing checkpoint file The Safetensors format is also supported. --- modules/sd_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 300387a9b..45c8b0c2d 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -158,7 +158,7 @@ def select_checkpoint(): print(f" - directory {model_path}", file=sys.stderr) if shared.cmd_opts.ckpt_dir is not None: print(f" - directory {os.path.abspath(shared.cmd_opts.ckpt_dir)}", file=sys.stderr) - print("Can't run without a checkpoint. Find and place a .ckpt file into any of those locations. The program will exit.", file=sys.stderr) + print("Can't run without a checkpoint. Find and place a .ckpt or .safetensors file into any of those locations. The program will exit.", file=sys.stderr) exit(1) checkpoint_info = next(iter(checkpoints_list.values())) From 269833067de1e7d0b6a6bd65724743d6b88a133f Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 2 Feb 2023 09:37:01 -0500 Subject: [PATCH 14/72] instruct-pix2pix support --- modules/processing.py | 2 +- modules/sd_samplers_kdiffusion.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index e544c2e16..f299e04da 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -186,7 +186,7 @@ class StableDiffusionProcessing: return conditioning def edit_image_conditioning(self, source_image): - conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image)) + conditioning_image = self.sd_model.encode_first_stage(source_image).mode() return conditioning_image diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index aa7f106b3..31ee22d3f 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -77,9 +77,9 @@ class CFGDenoiser(torch.nn.Module): batch_size = len(conds_list) repeats = [len(conds_list[i]) for i in range(batch_size)] - x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) - sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [image_cond]) denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) cfg_denoiser_callback(denoiser_params) @@ -88,7 +88,7 @@ class CFGDenoiser(torch.nn.Module): sigma_in = denoiser_params.sigma if tensor.shape[1] == uncond.shape[1]: - cond_in = torch.cat([tensor, uncond]) + cond_in = torch.cat([tensor, uncond, uncond]) if shared.batch_cond_uncond: x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) From cf0cfefe910b0de18c4751ce8d8cf7a6053a39b0 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 2 Feb 2023 19:15:38 -0500 Subject: [PATCH 15/72] Revert "instruct-pix2pix support" This reverts commit 269833067de1e7d0b6a6bd65724743d6b88a133f. --- modules/processing.py | 2 +- modules/sd_samplers_kdiffusion.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index f299e04da..e544c2e16 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -186,7 +186,7 @@ class StableDiffusionProcessing: return conditioning def edit_image_conditioning(self, source_image): - conditioning_image = self.sd_model.encode_first_stage(source_image).mode() + conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image)) return conditioning_image diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 31ee22d3f..aa7f106b3 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -77,9 +77,9 @@ class CFGDenoiser(torch.nn.Module): batch_size = len(conds_list) repeats = [len(conds_list[i]) for i in range(batch_size)] - x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) - sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [image_cond]) + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) cfg_denoiser_callback(denoiser_params) @@ -88,7 +88,7 @@ class CFGDenoiser(torch.nn.Module): sigma_in = denoiser_params.sigma if tensor.shape[1] == uncond.shape[1]: - cond_in = torch.cat([tensor, uncond, uncond]) + cond_in = torch.cat([tensor, uncond]) if shared.batch_cond_uncond: x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) From 3b2ad20ac1753cb664bd8954dd34f0c04d3678c2 Mon Sep 17 00:00:00 2001 From: Kyle Date: Thu, 2 Feb 2023 19:19:45 -0500 Subject: [PATCH 16/72] Processing only, no CFGDenoiser change Allows instruct-pix2pix --- modules/processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/processing.py b/modules/processing.py index e544c2e16..f299e04da 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -186,7 +186,7 @@ class StableDiffusionProcessing: return conditioning def edit_image_conditioning(self, source_image): - conditioning_image = self.sd_model.get_first_stage_encoding(self.sd_model.encode_first_stage(source_image)) + conditioning_image = self.sd_model.encode_first_stage(source_image).mode() return conditioning_image From 982295aee53dd4208fca1c13aa726f943806e790 Mon Sep 17 00:00:00 2001 From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com> Date: Sat, 4 Feb 2023 01:50:38 +0300 Subject: [PATCH 17/72] Fix img2imgalt after samplers separation --- scripts/img2imgalt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index cbdfc6b3c..2572443f9 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -6,7 +6,7 @@ from tqdm import trange import modules.scripts as scripts import gradio as gr -from modules import processing, shared, sd_samplers, prompt_parser +from modules import processing, shared, sd_samplers, prompt_parser, sd_samplers_common from modules.processing import Processed from modules.shared import opts, cmd_opts, state @@ -50,7 +50,7 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): x = x + d * dt - sd_samplers.store_latent(x) + sd_samplers_common.store_latent(x) # This shouldn't be necessary, but solved some VRAM issues del x_in, sigma_in, cond_in, c_out, c_in, t, @@ -104,7 +104,7 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): dt = sigmas[i] - sigmas[i - 1] x = x + d * dt - sd_samplers.store_latent(x) + sd_samplers_common.store_latent(x) # This shouldn't be necessary, but solved some VRAM issues del x_in, sigma_in, cond_in, c_out, c_in, t, From 6c6c6636bb123d664999c888cda47a1f8bad635b Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 3 Feb 2023 18:19:56 -0500 Subject: [PATCH 18/72] Image CFG Added (Full Implementation) Uses separate denoiser for edit (instruct-pix2pix) models No impact to txt2img or regular img2img "Image CFG Scale" will only apply to instruct-pix2pix models and metadata will only be added if using such model --- modules/img2img.py | 3 +- modules/processing.py | 4 +- modules/sd_samplers_kdiffusion.py | 101 ++++++++++++++++++++++++++++-- modules/ui.py | 3 + 4 files changed, 103 insertions(+), 8 deletions(-) diff --git a/modules/img2img.py b/modules/img2img.py index f813299c9..bcc158dc9 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -76,7 +76,7 @@ def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args): processed_image.save(os.path.join(output_dir, filename)) -def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, img2img_batch_inpaint_mask_dir: str, override_settings_texts, *args): +def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, img2img_batch_inpaint_mask_dir: str, override_settings_texts, *args): override_settings = create_override_settings_dict(override_settings_texts) is_batch = mode == 5 @@ -142,6 +142,7 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s inpainting_fill=inpainting_fill, resize_mode=resize_mode, denoising_strength=denoising_strength, + image_cfg_scale=image_cfg_scale, inpaint_full_res=inpaint_full_res, inpaint_full_res_padding=inpaint_full_res_padding, inpainting_mask_invert=inpainting_mask_invert, diff --git a/modules/processing.py b/modules/processing.py index f299e04da..c33694cc8 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -445,6 +445,7 @@ def create_infotext(p, all_prompts, all_seeds, all_subseeds, comments=None, iter "Steps": p.steps, "Sampler": p.sampler_name, "CFG scale": p.cfg_scale, + "Image CFG scale": getattr(p, 'image_cfg_scale', None), "Seed": all_seeds[index], "Face restoration": (opts.face_restoration_model if p.restore_faces else None), "Size": f"{p.width}x{p.height}", @@ -901,12 +902,13 @@ class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): class StableDiffusionProcessingImg2Img(StableDiffusionProcessing): sampler = None - def __init__(self, init_images: list = None, resize_mode: int = 0, denoising_strength: float = 0.75, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = True, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, **kwargs): + def __init__(self, init_images: list = None, resize_mode: int = 0, denoising_strength: float = 0.75, image_cfg_scale: float = None, mask: Any = None, mask_blur: int = 4, inpainting_fill: int = 0, inpaint_full_res: bool = True, inpaint_full_res_padding: int = 0, inpainting_mask_invert: int = 0, initial_noise_multiplier: float = None, **kwargs): super().__init__(**kwargs) self.init_images = init_images self.resize_mode: int = resize_mode self.denoising_strength: float = denoising_strength + self.image_cfg_scale: float = image_cfg_scale if shared.sd_model.cond_stage_key == "edit" else None self.init_latent = None self.image_mask = mask self.latent_mask = None diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index aa7f106b3..a16ba69be 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -1,6 +1,7 @@ from collections import deque import torch import inspect +import einops import k_diffusion.sampling from modules import prompt_parser, devices, sd_samplers_common @@ -40,6 +41,90 @@ sampler_extra_params = { 'sample_dpm_2': ['s_churn', 's_tmin', 's_tmax', 's_noise'], } +class CFGDenoiserEdit(torch.nn.Module): + """ + Classifier free guidance denoiser. A wrapper for stable diffusion model (specifically for unet) + that can take a noisy picture and produce a noise-free picture using two guidances (prompts) + instead of one. Originally, the second prompt is just an empty string, but we use non-empty + negative prompt. + """ + + def __init__(self, model): + super().__init__() + self.inner_model = model + self.mask = None + self.nmask = None + self.init_latent = None + self.step = 0 + + def combine_denoised(self, x_out, conds_list, uncond, cond_scale, image_cfg_scale): + denoised_uncond = x_out[-uncond.shape[0]:] + denoised = torch.clone(denoised_uncond) + + for i, conds in enumerate(conds_list): + for cond_index, weight in conds: + out_cond, out_img_cond, out_uncond = x_out.chunk(3) + denoised[i] = out_uncond[cond_index] + cond_scale * (out_cond[cond_index] - out_img_cond[cond_index]) + image_cfg_scale * (out_img_cond[cond_index] - out_uncond[cond_index]) + + return denoised + + def forward(self, x, sigma, uncond, cond, cond_scale, image_cond, image_cfg_scale): + if state.interrupted or state.skipped: + raise sd_samplers_common.InterruptedException + + conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step) + uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step) + + batch_size = len(conds_list) + repeats = [len(conds_list[i]) for i in range(batch_size)] + + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [torch.zeros_like(self.init_latent)]) + + denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) + cfg_denoiser_callback(denoiser_params) + x_in = denoiser_params.x + image_cond_in = denoiser_params.image_cond + sigma_in = denoiser_params.sigma + + if tensor.shape[1] == uncond.shape[1]: + cond_in = torch.cat([tensor, uncond, uncond]) + + if shared.batch_cond_uncond: + x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) + else: + x_out = torch.zeros_like(x_in) + for batch_offset in range(0, x_out.shape[0], batch_size): + a = batch_offset + b = a + batch_size + x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": [cond_in[a:b]], "c_concat": [image_cond_in[a:b]]}) + else: + x_out = torch.zeros_like(x_in) + batch_size = batch_size*2 if shared.batch_cond_uncond else batch_size + for batch_offset in range(0, tensor.shape[0], batch_size): + a = batch_offset + b = min(a + batch_size, tensor.shape[0]) + x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": torch.cat([tensor[a:b]], uncond) , "c_concat": [image_cond_in[a:b]]}) + + x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]}) + + devices.test_for_nans(x_out, "unet") + + if opts.live_preview_content == "Prompt": + sd_samplers_common.store_latent(x_out[0:uncond.shape[0]]) + elif opts.live_preview_content == "Negative prompt": + sd_samplers_common.store_latent(x_out[-uncond.shape[0]:]) + + denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale, image_cfg_scale) + + if self.mask is not None: + denoised = self.init_latent * self.mask + self.nmask * denoised + + self.step += 1 + + return denoised + class CFGDenoiser(torch.nn.Module): """ @@ -78,8 +163,8 @@ class CFGDenoiser(torch.nn.Module): repeats = [len(conds_list[i]) for i in range(batch_size)] x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) cfg_denoiser_callback(denoiser_params) @@ -160,7 +245,7 @@ class KDiffusionSampler: self.funcname = funcname self.func = getattr(k_diffusion.sampling, self.funcname) self.extra_params = sampler_extra_params.get(funcname, []) - self.model_wrap_cfg = CFGDenoiser(self.model_wrap) + self.model_wrap_cfg = CFGDenoiser(self.model_wrap) if not shared.sd_model.cond_stage_key == "edit" else CFGDenoiserEdit(self.model_wrap) self.sampler_noises = None self.stop_at = None self.eta = None @@ -260,13 +345,17 @@ class KDiffusionSampler: self.model_wrap_cfg.init_latent = x self.last_latent = x - - samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args={ + extra_args={ 'cond': conditioning, 'image_cond': image_conditioning, 'uncond': unconditional_conditioning, - 'cond_scale': p.cfg_scale - }, disable=False, callback=self.callback_state, **extra_params_kwargs)) + 'cond_scale': p.cfg_scale, + } + + if p.image_cfg_scale: + extra_args['image_cfg_scale'] = p.image_cfg_scale + + samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args=extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs)) return samples diff --git a/modules/ui.py b/modules/ui.py index 5e34fb074..f2f7de8b9 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -766,6 +766,7 @@ def create_ui(): elif category == "cfg": with FormGroup(): cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=7.0, elem_id="img2img_cfg_scale") + image_cfg_scale = gr.Slider(minimum=0, maximum=3.0, step=0.05, label='Image CFG Scale (for instruct-pix2pix models only)', value=1.5, elem_id="img2img_image_cfg_scale") denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength") elif category == "seed": @@ -861,6 +862,7 @@ def create_ui(): batch_count, batch_size, cfg_scale, + image_cfg_scale, denoising_strength, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, seed_checkbox, @@ -947,6 +949,7 @@ def create_ui(): (sampler_index, "Sampler"), (restore_faces, "Face restoration"), (cfg_scale, "CFG scale"), + (image_cfg_scale, "Image CFG scale"), (seed, "Seed"), (width, "Size-1"), (height, "Size-2"), From c27c0de0f73c5f533acfa10426dbac7ac988bc85 Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 3 Feb 2023 19:15:32 -0500 Subject: [PATCH 19/72] txt2img Hires Fix --- modules/processing.py | 1 + modules/sd_samplers_kdiffusion.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/processing.py b/modules/processing.py index c33694cc8..e1b53ac0a 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -268,6 +268,7 @@ class Processed: self.height = p.height self.sampler_name = p.sampler_name self.cfg_scale = p.cfg_scale + self.image_cfg_scale = getattr(p, 'image_cfg_scale', None) self.steps = p.steps self.batch_size = p.batch_size self.restore_faces = p.restore_faces diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index a16ba69be..6107e99e7 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -352,7 +352,7 @@ class KDiffusionSampler: 'cond_scale': p.cfg_scale, } - if p.image_cfg_scale: + if hasattr(p, 'image_cfg_scale'): extra_args['image_cfg_scale'] = p.image_cfg_scale samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args=extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs)) From ba6a4e7e9431d02ba3656c6ae44d5dfe29908d68 Mon Sep 17 00:00:00 2001 From: Kyle Date: Fri, 3 Feb 2023 19:46:13 -0500 Subject: [PATCH 20/72] Use original CFGDenoiser if image_cfg_scale = 1 If image_cfg_scale is =1 then the original image is not used for the output. We can then use the original CFGDenoiser to get the same result to support AND functionality. Maybe in the future AND can be supported with "Image CFG Scale" --- modules/sd_samplers_kdiffusion.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 6107e99e7..6c57fdec8 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -245,7 +245,7 @@ class KDiffusionSampler: self.funcname = funcname self.func = getattr(k_diffusion.sampling, self.funcname) self.extra_params = sampler_extra_params.get(funcname, []) - self.model_wrap_cfg = CFGDenoiser(self.model_wrap) if not shared.sd_model.cond_stage_key == "edit" else CFGDenoiserEdit(self.model_wrap) + self.model_wrap_cfg = CFGDenoiser(self.model_wrap) self.sampler_noises = None self.stop_at = None self.eta = None @@ -280,6 +280,9 @@ class KDiffusionSampler: return p.steps def initialize(self, p): + if shared.sd_model.cond_stage_key == "edit" and getattr(p, 'image_cfg_scale', None) != 1: + self.model_wrap_cfg = CFGDenoiserEdit(self.model_wrap) + self.model_wrap_cfg.mask = p.mask if hasattr(p, 'mask') else None self.model_wrap_cfg.nmask = p.nmask if hasattr(p, 'nmask') else None self.model_wrap_cfg.step = 0 @@ -352,7 +355,7 @@ class KDiffusionSampler: 'cond_scale': p.cfg_scale, } - if hasattr(p, 'image_cfg_scale'): + if hasattr(p, 'image_cfg_scale') and p.image_cfg_scale != 1 and p.image_cfg_scale != None: extra_args['image_cfg_scale'] = p.image_cfg_scale samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args=extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs)) From 4306659c4dab1a2ae611ac2a7487b87e1c513adf Mon Sep 17 00:00:00 2001 From: brkirch Date: Sat, 4 Feb 2023 01:22:06 -0500 Subject: [PATCH 21/72] Remove unused code --- modules/mac_specific.py | 3 --- modules/shared.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/modules/mac_specific.py b/modules/mac_specific.py index e39d670eb..ddcea53b9 100644 --- a/modules/mac_specific.py +++ b/modules/mac_specific.py @@ -4,9 +4,6 @@ from modules.sd_hijack_utils import CondFunc from packaging import version -device = None - - # has_mps is only available in nightly pytorch (for now) and macOS 12.3+. # check `getattr` and try it for compatibility def check_for_mps() -> bool: diff --git a/modules/shared.py b/modules/shared.py index 59f12cd8d..5600d480c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -145,9 +145,6 @@ devices.device, devices.device_interrogate, devices.device_gfpgan, devices.devic (devices.cpu if any(y in cmd_opts.use_cpu for y in [x, 'all']) else devices.get_optimal_device() for x in ['sd', 'interrogate', 'gfpgan', 'esrgan', 'codeformer']) device = devices.device -if sys.platform == "darwin": - from modules import mac_specific - mac_specific.device = device weight_load_location = None if cmd_opts.lowram else "cpu" batch_cond_uncond = cmd_opts.always_batch_cond_uncond or not (cmd_opts.lowvram or cmd_opts.medvram) From 72dd5785d9721b95e8d61210a56be8f6c6b1e97d Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 11:06:17 +0300 Subject: [PATCH 22/72] merge CFGDenoiserEdit and CFGDenoiser into single object --- modules/sd_samplers_kdiffusion.py | 133 +++++++++--------------------- 1 file changed, 37 insertions(+), 96 deletions(-) diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 6c57fdec8..f076fc550 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -41,90 +41,6 @@ sampler_extra_params = { 'sample_dpm_2': ['s_churn', 's_tmin', 's_tmax', 's_noise'], } -class CFGDenoiserEdit(torch.nn.Module): - """ - Classifier free guidance denoiser. A wrapper for stable diffusion model (specifically for unet) - that can take a noisy picture and produce a noise-free picture using two guidances (prompts) - instead of one. Originally, the second prompt is just an empty string, but we use non-empty - negative prompt. - """ - - def __init__(self, model): - super().__init__() - self.inner_model = model - self.mask = None - self.nmask = None - self.init_latent = None - self.step = 0 - - def combine_denoised(self, x_out, conds_list, uncond, cond_scale, image_cfg_scale): - denoised_uncond = x_out[-uncond.shape[0]:] - denoised = torch.clone(denoised_uncond) - - for i, conds in enumerate(conds_list): - for cond_index, weight in conds: - out_cond, out_img_cond, out_uncond = x_out.chunk(3) - denoised[i] = out_uncond[cond_index] + cond_scale * (out_cond[cond_index] - out_img_cond[cond_index]) + image_cfg_scale * (out_img_cond[cond_index] - out_uncond[cond_index]) - - return denoised - - def forward(self, x, sigma, uncond, cond, cond_scale, image_cond, image_cfg_scale): - if state.interrupted or state.skipped: - raise sd_samplers_common.InterruptedException - - conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step) - uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step) - - batch_size = len(conds_list) - repeats = [len(conds_list[i]) for i in range(batch_size)] - - x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) - sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [torch.zeros_like(self.init_latent)]) - - denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) - cfg_denoiser_callback(denoiser_params) - x_in = denoiser_params.x - image_cond_in = denoiser_params.image_cond - sigma_in = denoiser_params.sigma - - if tensor.shape[1] == uncond.shape[1]: - cond_in = torch.cat([tensor, uncond, uncond]) - - if shared.batch_cond_uncond: - x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) - else: - x_out = torch.zeros_like(x_in) - for batch_offset in range(0, x_out.shape[0], batch_size): - a = batch_offset - b = a + batch_size - x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": [cond_in[a:b]], "c_concat": [image_cond_in[a:b]]}) - else: - x_out = torch.zeros_like(x_in) - batch_size = batch_size*2 if shared.batch_cond_uncond else batch_size - for batch_offset in range(0, tensor.shape[0], batch_size): - a = batch_offset - b = min(a + batch_size, tensor.shape[0]) - x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": torch.cat([tensor[a:b]], uncond) , "c_concat": [image_cond_in[a:b]]}) - - x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]}) - - devices.test_for_nans(x_out, "unet") - - if opts.live_preview_content == "Prompt": - sd_samplers_common.store_latent(x_out[0:uncond.shape[0]]) - elif opts.live_preview_content == "Negative prompt": - sd_samplers_common.store_latent(x_out[-uncond.shape[0]:]) - - denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale, image_cfg_scale) - - if self.mask is not None: - denoised = self.init_latent * self.mask + self.nmask * denoised - - self.step += 1 - - return denoised - class CFGDenoiser(torch.nn.Module): """ @@ -141,6 +57,7 @@ class CFGDenoiser(torch.nn.Module): self.nmask = None self.init_latent = None self.step = 0 + self.image_cfg_scale = None def combine_denoised(self, x_out, conds_list, uncond, cond_scale): denoised_uncond = x_out[-uncond.shape[0]:] @@ -152,19 +69,36 @@ class CFGDenoiser(torch.nn.Module): return denoised + def combine_denoised_for_edit_model(self, x_out, cond_scale): + out_cond, out_img_cond, out_uncond = x_out.chunk(3) + denoised = out_uncond + cond_scale * (out_cond - out_img_cond) + self.image_cfg_scale * (out_img_cond - out_uncond) + + return denoised + def forward(self, x, sigma, uncond, cond, cond_scale, image_cond): if state.interrupted or state.skipped: raise sd_samplers_common.InterruptedException + # at self.image_cfg_scale == 1.0 produced results for edit model are the same as with normal sampling, + # so is_edit_model is set to False to support AND composition. + is_edit_model = shared.sd_model.cond_stage_key == "edit" and self.image_cfg_scale is not None and self.image_cfg_scale != 1.0 + conds_list, tensor = prompt_parser.reconstruct_multicond_batch(cond, self.step) uncond = prompt_parser.reconstruct_cond_batch(uncond, self.step) + assert not is_edit_model or all([len(conds) == 1 for conds in conds_list]), "AND is not supported for InstructPix2Pix checkpoint (unless using Image CFG scale = 1.0)" + batch_size = len(conds_list) repeats = [len(conds_list[i]) for i in range(batch_size)] - x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) - sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) - image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) + if not is_edit_model: + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond]) + else: + x_in = torch.cat([torch.stack([x[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [x] + [x]) + sigma_in = torch.cat([torch.stack([sigma[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [sigma] + [sigma]) + image_cond_in = torch.cat([torch.stack([image_cond[i] for _ in range(n)]) for i, n in enumerate(repeats)] + [image_cond] + [torch.zeros_like(self.init_latent)]) denoiser_params = CFGDenoiserParams(x_in, image_cond_in, sigma_in, state.sampling_step, state.sampling_steps) cfg_denoiser_callback(denoiser_params) @@ -173,7 +107,10 @@ class CFGDenoiser(torch.nn.Module): sigma_in = denoiser_params.sigma if tensor.shape[1] == uncond.shape[1]: - cond_in = torch.cat([tensor, uncond]) + if not is_edit_model: + cond_in = torch.cat([tensor, uncond]) + else: + cond_in = torch.cat([tensor, uncond, uncond]) if shared.batch_cond_uncond: x_out = self.inner_model(x_in, sigma_in, cond={"c_crossattn": [cond_in], "c_concat": [image_cond_in]}) @@ -189,7 +126,13 @@ class CFGDenoiser(torch.nn.Module): for batch_offset in range(0, tensor.shape[0], batch_size): a = batch_offset b = min(a + batch_size, tensor.shape[0]) - x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": [tensor[a:b]], "c_concat": [image_cond_in[a:b]]}) + + if not is_edit_model: + c_crossattn = [tensor[a:b]] + else: + c_crossattn = torch.cat([tensor[a:b]], uncond) + + x_out[a:b] = self.inner_model(x_in[a:b], sigma_in[a:b], cond={"c_crossattn": c_crossattn, "c_concat": [image_cond_in[a:b]]}) x_out[-uncond.shape[0]:] = self.inner_model(x_in[-uncond.shape[0]:], sigma_in[-uncond.shape[0]:], cond={"c_crossattn": [uncond], "c_concat": [image_cond_in[-uncond.shape[0]:]]}) @@ -200,7 +143,10 @@ class CFGDenoiser(torch.nn.Module): elif opts.live_preview_content == "Negative prompt": sd_samplers_common.store_latent(x_out[-uncond.shape[0]:]) - denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale) + if not is_edit_model: + denoised = self.combine_denoised(x_out, conds_list, uncond, cond_scale) + else: + denoised = self.combine_denoised_for_edit_model(x_out, cond_scale) if self.mask is not None: denoised = self.init_latent * self.mask + self.nmask * denoised @@ -280,12 +226,10 @@ class KDiffusionSampler: return p.steps def initialize(self, p): - if shared.sd_model.cond_stage_key == "edit" and getattr(p, 'image_cfg_scale', None) != 1: - self.model_wrap_cfg = CFGDenoiserEdit(self.model_wrap) - self.model_wrap_cfg.mask = p.mask if hasattr(p, 'mask') else None self.model_wrap_cfg.nmask = p.nmask if hasattr(p, 'nmask') else None self.model_wrap_cfg.step = 0 + self.model_wrap_cfg.image_cfg_scale = getattr(p, 'image_cfg_scale', None) self.eta = p.eta if p.eta is not None else opts.eta_ancestral k_diffusion.sampling.torch = TorchHijack(self.sampler_noises if self.sampler_noises is not None else []) @@ -355,9 +299,6 @@ class KDiffusionSampler: 'cond_scale': p.cfg_scale, } - if hasattr(p, 'image_cfg_scale') and p.image_cfg_scale != 1 and p.image_cfg_scale != None: - extra_args['image_cfg_scale'] = p.image_cfg_scale - samples = self.launch_sampling(t_enc + 1, lambda: self.func(self.model_wrap_cfg, xi, extra_args=extra_args, disable=False, callback=self.callback_state, **extra_params_kwargs)) return samples From c4b9ed1a2791e411f95a96a6324b4986b8b85b84 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 11:18:44 +0300 Subject: [PATCH 23/72] make Image CFG Scale only show if instrutpix2pix model is loaded --- modules/ui.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/ui.py b/modules/ui.py index f2f7de8b9..f5df1ffeb 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -765,8 +765,9 @@ def create_ui(): elif category == "cfg": with FormGroup(): - cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=7.0, elem_id="img2img_cfg_scale") - image_cfg_scale = gr.Slider(minimum=0, maximum=3.0, step=0.05, label='Image CFG Scale (for instruct-pix2pix models only)', value=1.5, elem_id="img2img_image_cfg_scale") + with FormRow(): + cfg_scale = gr.Slider(minimum=1.0, maximum=30.0, step=0.5, label='CFG Scale', value=7.0, elem_id="img2img_cfg_scale") + image_cfg_scale = gr.Slider(minimum=0, maximum=3.0, step=0.05, label='Image CFG Scale', value=1.5, elem_id="img2img_image_cfg_scale", visible=shared.sd_model and shared.sd_model.cond_stage_key == "edit") denoising_strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Denoising strength', value=0.75, elem_id="img2img_denoising_strength") elif category == "seed": @@ -1594,6 +1595,12 @@ def create_ui(): outputs=[component, text_settings], ) + text_settings.change( + fn=lambda: gr.update(visible=shared.sd_model and shared.sd_model.cond_stage_key == "edit"), + inputs=[], + outputs=[image_cfg_scale], + ) + button_set_checkpoint = gr.Button('Change checkpoint', elem_id='change_checkpoint', visible=False) button_set_checkpoint.click( fn=lambda value, _: run_settings_single(value, key='sd_model_checkpoint'), From 81823407d9b3c3daf2f9de59e0d75ef9a257f902 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 11:38:56 +0300 Subject: [PATCH 24/72] add --no-hashing --- modules/hashes.py | 4 ++++ modules/hypernetworks/hypernetwork.py | 2 +- modules/sd_models.py | 3 +++ modules/shared.py | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/hashes.py b/modules/hashes.py index 819362a3f..83272a078 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -4,6 +4,7 @@ import os.path import filelock +from modules import shared from modules.paths import data_path @@ -68,6 +69,9 @@ def sha256(filename, title): if sha256_value is not None: return sha256_value + if shared.cmd_opts.no_hashing: + return None + print(f"Calculating sha256 for {filename}: ", end='') sha256_value = calculate_sha256(filename) print(f"{sha256_value}") diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 503534e24..825a93b28 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -307,7 +307,7 @@ class Hypernetwork: def shorthash(self): sha256 = hashes.sha256(self.filename, f'hypernet/{self.name}') - return sha256[0:10] + return sha256[0:10] if sha256 else None def list_hypernetworks(path): diff --git a/modules/sd_models.py b/modules/sd_models.py index 300387a9b..6c6bb5712 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -59,6 +59,9 @@ class CheckpointInfo: def calculate_shorthash(self): self.sha256 = hashes.sha256(self.filename, "checkpoint/" + self.name) + if self.sha256 is None: + return + self.shorthash = self.sha256[0:10] if self.shorthash not in self.ids: diff --git a/modules/shared.py b/modules/shared.py index 5600d480c..79fbf7249 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -106,7 +106,7 @@ parser.add_argument("--tls-certfile", type=str, help="Partially enables TLS, req parser.add_argument("--server-name", type=str, help="Sets hostname of server", default=None) parser.add_argument("--gradio-queue", action='store_true', help="Uses gradio queue; experimental option; breaks restart UI button") parser.add_argument("--skip-version-check", action='store_true', help="Do not check versions of torch and xformers") - +parser.add_argument("--no-hashing", action='store_true', help="disable sha256 hashing of checkpoints to help loading performance", default=False) script_loading.preload_extensions(extensions.extensions_dir, parser) From 40e51fd6efa9c09a82c5ab391dbbd2c806971582 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 13:28:53 +0300 Subject: [PATCH 25/72] add margin parameter to draw_grid_annotations --- modules/images.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/images.py b/modules/images.py index ae3cdaf4a..4bdbd730d 100644 --- a/modules/images.py +++ b/modules/images.py @@ -130,7 +130,7 @@ class GridAnnotation: self.size = None -def draw_grid_annotations(im, width, height, hor_texts, ver_texts): +def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0): def wrap(drawing, text, font, line_length): lines = [''] for word in text.split(): @@ -194,25 +194,28 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts): line.allowed_width = allowed_width hor_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in hor_texts] - ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in - ver_texts] + ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in ver_texts] pad_top = 0 if sum(hor_text_heights) == 0 else max(hor_text_heights) + line_spacing * 2 - result = Image.new("RGB", (im.width + pad_left, im.height + pad_top), "white") - result.paste(im, (pad_left, pad_top)) + result = Image.new("RGB", (im.width + pad_left + margin * (rows-1), im.height + pad_top + margin * (cols-1)), "white") + + for row in range(rows): + for col in range(cols): + cell = im.crop((width * col, height * row, width * (col+1), height * (row+1))) + result.paste(cell, (pad_left + (width + margin) * col, pad_top + (height + margin) * row)) d = ImageDraw.Draw(result) for col in range(cols): - x = pad_left + width * col + width / 2 + x = pad_left + (width + margin) * col + width / 2 y = pad_top / 2 - hor_text_heights[col] / 2 draw_texts(d, x, y, hor_texts[col], fnt, fontsize) for row in range(rows): x = pad_left / 2 - y = pad_top + height * row + height / 2 - ver_text_heights[row] / 2 + y = pad_top + (height + margin) * row + height / 2 - ver_text_heights[row] / 2 draw_texts(d, x, y, ver_texts[row], fnt, fontsize) From 3e0f9a75438fa815429b5530261bcf7d80f3f101 Mon Sep 17 00:00:00 2001 From: AUTOMATIC <16777216c@gmail.com> Date: Sat, 4 Feb 2023 15:23:16 +0300 Subject: [PATCH 26/72] fix issue with switching back to checkpoint that had its checksum calculated during runtime mentioned in #7506 --- modules/sd_models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 0e61d3233..af1731e51 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -65,10 +65,11 @@ class CheckpointInfo: self.shorthash = self.sha256[0:10] if self.shorthash not in self.ids: - self.ids += [self.shorthash, self.sha256] - self.register() + self.ids += [self.shorthash, self.sha256, f'{self.name} [{self.shorthash}]'] + checkpoints_list.pop(self.title) self.title = f'{self.name} [{self.shorthash}]' + self.register() return self.shorthash From dd20fc0fda5ac7cbb93af0b93222ee5a17f7705e Mon Sep 17 00:00:00 2001 From: w-e-w <40751091+w-e-w@users.noreply.github.com> Date: Sat, 4 Feb 2023 23:23:20 +0900 Subject: [PATCH 27/72] fix --help show correct help message --- launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch.py b/launch.py index 9fd766d1a..a68bb3a91 100644 --- a/launch.py +++ b/launch.py @@ -242,7 +242,7 @@ def prepare_environment(): sys.argv += shlex.split(commandline_args) - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--ui-settings-file", type=str, help="filename to use for ui settings", default='config.json') args, _ = parser.parse_known_args(sys.argv) From 6524478850ba1b285fee2593b113dfb726b0bd9f Mon Sep 17 00:00:00 2001 From: spezialspezial <75758219+spezialspezial@users.noreply.github.com> Date: Sat, 4 Feb 2023 16:52:15 +0100 Subject: [PATCH 28/72] Update modelloader.py os.path.getmtime(filename) throws exception later in codepath when meeting broken symlink. For now catch it here early but more checks could be added for robustness. --- modules/modelloader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/modelloader.py b/modules/modelloader.py index e9aa514eb..fc3f6249f 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -45,6 +45,9 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None full_path = file if os.path.isdir(full_path): continue + if os.path.islink(full_path) and not os.path.exists(full_path): + print(f"Skipping broken symlink: {full_path}") + continue if ext_blacklist is not None and any([full_path.endswith(x) for x in ext_blacklist]): continue if len(ext_filter) != 0: From 88a46e8427fbaa73eb37c9eaabbb62b8647a5f32 Mon Sep 17 00:00:00 2001 From: "Alex \"mcmonkey\" Goodwin" Date: Sat, 4 Feb 2023 09:10:00 -0800 Subject: [PATCH 29/72] fix symlinks in extra networks ui 'absolute' and 'resolve' are equivalent, but 'resolve' resolves symlinks (which is an obscure specialty behavior usually not wanted) whereas 'absolute' treats symlinks as folders (which is the expected behavior). This commit allows you to symlink folders within your models/embeddings/etc. dirs and have preview images load as expected without issue. --- modules/ui_extra_networks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 95b30f4ab..90abec0a5 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -26,7 +26,7 @@ def add_pages_to_demo(app): def fetch_file(filename: str = ""): from starlette.responses import FileResponse - if not any([Path(x).resolve() in Path(filename).resolve().parents for x in allowed_dirs]): + if not any([Path(x).absolute() in Path(filename).absolute().parents for x in allowed_dirs]): raise ValueError(f"File cannot be fetched: {filename}. Must be in one of directories registered by extra pages.") ext = os.path.splitext(filename)[1].lower() From 5a1b62e9f8048e20a9ff47df73b16f8a0b5e673c Mon Sep 17 00:00:00 2001 From: techneconn Date: Sun, 5 Feb 2023 15:48:51 +0900 Subject: [PATCH 30/72] Add prompt_hash option for file/dir name pattern --- javascript/hints.js | 4 ++-- modules/images.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/javascript/hints.js b/javascript/hints.js index 75792d0dd..9aa82f246 100644 --- a/javascript/hints.js +++ b/javascript/hints.js @@ -66,8 +66,8 @@ titles = { "Interrogate": "Reconstruct prompt from existing image and put it into the prompt field.", - "Images filename pattern": "Use following tags to define how filenames for images are chosen: [steps], [cfg], [prompt], [prompt_no_styles], [prompt_spaces], [width], [height], [styles], [sampler], [seed], [model_hash], [model_name], [prompt_words], [date], [datetime], [datetime], [datetime