Skip to content

Commit

Permalink
Add dynamic denoising and inpaint bbox sizing (#678)
Browse files Browse the repository at this point in the history 
* Added dynamic denoising and inpaint bbox sizing

* Dynamic denoising: Once bboxes are available from the predictor, it is possible to calculate the size of the crop region relative to the original image size. Using this value, we can modulate the "Inpaint denoising strength" based on the region size, with smaller regions getting higher denoising, and smaller areas less.
* Several algorithms were tested, ultimately, a configurable power value worked best. Values between 2-4 are recommended (1 is equivalent to linear).

* Try match inpaint/bbox size: Again, using bbox sizes, we can determine more optimal dimensions and aspect ratio for the inpaint width and height.
* Only active for SDXL, as the model natively handles various dimensions and aspect ratios.

* Don't use inpaint/bbox matching if user has specified their own width and height

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Remove math.isclose.

* Remove math import

* Remove unneeded formatting

* Better descriptions for new features in settings.

* Tidy up bbox matching, filter out more resolutions earlier

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add strict and free inpaint bbox size matching

* Strict: SDXL only, same as original implementation
* Free (prefer smaller or larger): Theoretically works with any model. Adjusts the inpaint region to match the aspect ratio of the bbox exactly, favouring either the smaller dimension or larger dimension of the original inpaint region. We also round up (if needed) to the closest 8 pixels to make the dimensions nicer to diffusion/upscalers. "Prefer smaller" is the better option, as it will usually very closely match the original inpaint sizes.
* Also added a threshold to the difference between the original inpaint size and adjusted size, and ignore the adjusted size if it's very similar.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Use or for checking thresholds on new inpaint dimensions

* Rework free mode to a single setting

Should now always pick optimal dimensions

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
@LoganBooker @pre-commit-ci
LoganBooker and pre-commit-ci[bot] authored Aug 18, 2024
1 parent d3a38aa commit 38db25b
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 0 deletions.
5 changes: 5 additions & 0 deletions adetailer/args.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,11 @@ def need_skip(self) -> bool:
"Position (center to edge)",
"Area (large to small)",
]
INPAINT_BBOX_MATCH_MODES = [
"Off",
"Strict (SDXL only)",
"Free",
]
MASK_MERGE_INVERT = ["None", "Merge", "Merge and Invert"]

_script_default = (
Expand Down
135 changes: 135 additions & 0 deletions scripts/!adetailer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
from adetailer.args import (
BBOX_SORTBY,
BUILTIN_SCRIPT,
INPAINT_BBOX_MATCH_MODES,
SCRIPT_DEFAULT,
ADetailerArgs,
SkipImg2ImgOrig,
Expand Down Expand Up @@ -668,6 +669,103 @@ def get_image_mask(p) -> Image.Image:
width, height = p.width, p.height
return images.resize_image(p.resize_mode, mask, width, height)

@staticmethod
def get_dynamic_denoise_strength(denoise_strength, bbox, image):
denoise_power = opts.data.get("ad_dynamic_denoise_power", 0)
if denoise_power == 0:
return denoise_strength

image_pixels = image.width * image.height
bbox_pixels = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])

normalized_area = bbox_pixels / image_pixels
denoise_modifier = (1.0 - normalized_area) ** denoise_power

print(
f"[-] ADetailer: dynamic denoising -- {denoise_modifier:.2f} * {denoise_strength:.2f} = {denoise_strength * denoise_modifier:.2f}"
)

return denoise_strength * denoise_modifier

@staticmethod
def get_optimal_crop_image_size(inpaint_width, inpaint_height, bbox):
calculate_optimal_crop = opts.data.get("ad_match_inpaint_bbox_size", "Off")
if calculate_optimal_crop == "Off":
return (inpaint_width, inpaint_height)

optimal_resolution = None

bbox_width = bbox[2] - bbox[0]
bbox_height = bbox[3] - bbox[1]
bbox_aspect_ratio = bbox_width / bbox_height

if calculate_optimal_crop == "Strict (SDXL only)":
if not shared.sd_model.is_sdxl:
msg = "[-] ADetailer: strict inpaint bounding box size matching is only available for SDXL. Use Free mode instead."
print(msg)
return (inpaint_width, inpaint_height)

# Limit resolutions to those SDXL was trained on.
resolutions = [
(1024, 1024),
(1152, 896),
(896, 1152),
(1216, 832),
(832, 1216),
(1344, 768),
(768, 1344),
(1536, 640),
(640, 1536),
]

# Filter resolutions smaller than bbox, and any that could result in a total pixel size smaller than the current inpaint dimensions.
resolutions = [
res
for res in resolutions
if (res[0] >= bbox_width and res[1] >= bbox_height)
and (res[0] >= inpaint_width or res[1] >= inpaint_height)
]

if not resolutions:
return (inpaint_width, inpaint_height)

optimal_resolution = min(
resolutions,
key=lambda res: abs((res[0] / res[1]) - bbox_aspect_ratio),
)
elif calculate_optimal_crop == "Free":
scale_size = max(inpaint_width, inpaint_height)

if bbox_aspect_ratio > 1:
optimal_width = scale_size
optimal_height = scale_size / bbox_aspect_ratio
else:
optimal_width = scale_size * bbox_aspect_ratio
optimal_height = scale_size

# Round up to the nearest multiple of 8 to make the dimensions friendly for upscaling/diffusion.
optimal_width = ((optimal_width + 8 - 1) // 8) * 8
optimal_height = ((optimal_height + 8 - 1) // 8) * 8

optimal_resolution = (int(optimal_width), int(optimal_height))
else:
msg = "[-] ADetailer: unsupported inpaint bounding box match mode. Original inpainting dimensions will be used."
print(msg)

if optimal_resolution is None:
return (inpaint_width, inpaint_height)

# Only use optimal dimensions if they're different enough to current inpaint dimensions.
if (
abs(optimal_resolution[0] - inpaint_width) > inpaint_width * 0.1
or abs(optimal_resolution[1] - inpaint_height) > inpaint_height * 0.1
):
print(
f"[-] ADetailer: inpaint dimensions optimized -- {inpaint_width}x{inpaint_height} -> {optimal_resolution[0]}x{optimal_resolution[1]}"
)

return optimal_resolution

@rich_traceback
def process(self, p, *args_):
if getattr(p, "_ad_disabled", False):
Expand Down Expand Up @@ -773,6 +871,17 @@ def _postprocess_image_inner(

p2.cached_c = [None, None]
p2.cached_uc = [None, None]

p2.denoising_strength = self.get_dynamic_denoise_strength(
p2.denoising_strength, pred.bboxes[j], pp.image
)

# Don't override user-defined dimensions.
if not args.ad_use_inpaint_width_height:
p2.width, p2.height = self.get_optimal_crop_image_size(
p2.width, p2.height, pred.bboxes[j]
)

try:
processed = process_images(p2)
except NansException as e:
Expand Down Expand Up @@ -915,6 +1024,32 @@ def on_ui_settings():
),
)

shared.opts.add_option(
"ad_dynamic_denoise_power",
shared.OptionInfo(
default=0,
label="Power scaling for dynamic denoise strength based on bounding box size",
component=gr.Slider,
component_args={"minimum": -10, "maximum": 10, "step": 0.01},
section=section,
).info(
"Smaller areas get higher denoising, larger areas less. Maximum denoise strength is set by 'Inpaint denoising strength'. 0 = disabled; 1 = linear; 2-4 = recommended"
),
)

shared.opts.add_option(
"ad_match_inpaint_bbox_size",
shared.OptionInfo(
default="Off",
component=gr.Radio,
component_args={"choices": INPAINT_BBOX_MATCH_MODES},
label="Try to match inpainting size to bounding box size, if 'Use separate width/height' is not set",
section=section,
).info(
"Strict is for SDXL only, and matches exactly to trained SDXL resolutions. Free works with any model, but will use potentially unsupported dimensions."
),
)


# xyz_grid

Expand Down

0 comments on commit 38db25b

Please sign in to comment.