From 7dbf3960221dbfe249c81d36b5afc96b8586a51f Mon Sep 17 00:00:00 2001 From: limiteinductive Date: Tue, 12 Dec 2023 15:07:18 +0100 Subject: [PATCH] change the fl.Slincing interface --- src/refiners/fluxion/layers/basics.py | 12 +++++++----- .../foundationals/latent_diffusion/auto_encoder.py | 2 +- .../foundationals/latent_diffusion/image_prompt.py | 10 ++++++---- .../stable_diffusion_1/controlnet.py | 2 +- .../foundationals/segment_anything/mask_decoder.py | 8 ++++---- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/refiners/fluxion/layers/basics.py b/src/refiners/fluxion/layers/basics.py index 07e4aec6a..3c9f01992 100644 --- a/src/refiners/fluxion/layers/basics.py +++ b/src/refiners/fluxion/layers/basics.py @@ -84,16 +84,18 @@ def forward(self, x: Tensor) -> Tensor: class Slicing(Module): - def __init__(self, dim: int, start: int, length: int) -> None: + def __init__(self, dim: int = 0, start: int = 0, end: int = -1, step: int = 1) -> None: super().__init__() self.dim = dim self.start = start - self.length = length + self.end = end + self.step = step def forward(self, x: Tensor) -> Tensor: - if self.length < 0: - return x.narrow(self.dim, self.start, x.shape[self.dim] - self.start + self.length + 1) - return x.narrow(self.dim, self.start, self.length) + start = self.start if self.start >= 0 else x.shape[self.dim] + self.start + end = self.end if self.end >= 0 else x.shape[self.dim] + self.end + indices = torch.arange(start=start, end=end, step=self.step, device=x.device) + return x.index_select(self.dim, indices) class Squeeze(Module): diff --git a/src/refiners/foundationals/latent_diffusion/auto_encoder.py b/src/refiners/foundationals/latent_diffusion/auto_encoder.py index 2dc3bd5b1..9fee47d92 100644 --- a/src/refiners/foundationals/latent_diffusion/auto_encoder.py +++ b/src/refiners/foundationals/latent_diffusion/auto_encoder.py @@ -113,7 +113,7 @@ def __init__(self, device: Device | str | None = None, dtype: DType | None = Non ), Chain( Conv2d(in_channels=8, out_channels=8, kernel_size=1, device=device, dtype=dtype), - Slicing(dim=1, start=0, length=4), + Slicing(dim=1, end=4), ), ) diff --git a/src/refiners/foundationals/latent_diffusion/image_prompt.py b/src/refiners/foundationals/latent_diffusion/image_prompt.py index c25b82b8a..66c94e284 100644 --- a/src/refiners/foundationals/latent_diffusion/image_prompt.py +++ b/src/refiners/foundationals/latent_diffusion/image_prompt.py @@ -264,11 +264,11 @@ def __init__( InjectionPoint(), # Wq fl.Parallel( fl.Chain( - fl.Slicing(dim=1, start=0, length=text_sequence_length), + fl.Slicing(dim=1, end=text_sequence_length), InjectionPoint(), # Wk ), fl.Chain( - fl.Slicing(dim=1, start=text_sequence_length, length=image_sequence_length), + fl.Slicing(dim=1, start=text_sequence_length, end=image_sequence_length), fl.Linear( in_features=self.target.key_embedding_dim, out_features=self.target.inner_dim, @@ -280,11 +280,13 @@ def __init__( ), fl.Parallel( fl.Chain( - fl.Slicing(dim=1, start=0, length=text_sequence_length), + fl.Slicing(dim=1, end=text_sequence_length), InjectionPoint(), # Wv ), fl.Chain( - fl.Slicing(dim=1, start=text_sequence_length, length=image_sequence_length), + fl.Slicing( + dim=1, start=text_sequence_length, end=text_sequence_length + image_sequence_length + 1 + ), fl.Linear( in_features=self.target.key_embedding_dim, out_features=self.target.inner_dim, diff --git a/src/refiners/foundationals/latent_diffusion/stable_diffusion_1/controlnet.py b/src/refiners/foundationals/latent_diffusion/stable_diffusion_1/controlnet.py index 21e2f5023..1b8b9ff94 100644 --- a/src/refiners/foundationals/latent_diffusion/stable_diffusion_1/controlnet.py +++ b/src/refiners/foundationals/latent_diffusion/stable_diffusion_1/controlnet.py @@ -86,7 +86,7 @@ def __init__( self.scale = scale super().__init__( TimestepEncoder(context_key=f"timestep_embedding_{name}", device=device, dtype=dtype), - Slicing(dim=1, start=0, length=4), # support inpainting + Slicing(dim=1, end=4), # support inpainting DownBlocks(in_channels=4, device=device, dtype=dtype), MiddleBlock(device=device, dtype=dtype), ) diff --git a/src/refiners/foundationals/segment_anything/mask_decoder.py b/src/refiners/foundationals/segment_anything/mask_decoder.py index b0ee47def..b032eb4cc 100644 --- a/src/refiners/foundationals/segment_anything/mask_decoder.py +++ b/src/refiners/foundationals/segment_anything/mask_decoder.py @@ -60,7 +60,7 @@ def __init__( super().__init__( *[ fl.Chain( - fl.Slicing(dim=1, start=i + 1, length=1), + fl.Slicing(dim=1, start=i + 1, end=i + 2), fl.MultiLinear( input_dim=embedding_dim, output_dim=embedding_dim // 8, @@ -156,7 +156,7 @@ def __init__( ), other=DenseEmbeddingUpscaling(embedding_dim=embedding_dim, device=device, dtype=dtype), ), - fl.Slicing(dim=1, start=1, length=num_mask_tokens), + fl.Slicing(dim=1, start=1, end=num_mask_tokens + 1), fl.Reshape(num_mask_tokens, embedding_dim, embedding_dim), ) @@ -173,7 +173,7 @@ def __init__( self.embedding_dim = embedding_dim self.num_layers = num_layers super().__init__( - fl.Slicing(dim=1, start=0, length=1), + fl.Slicing(dim=1, start=0, end=2), fl.Squeeze(dim=0), fl.MultiLinear( input_dim=embedding_dim, @@ -183,7 +183,7 @@ def __init__( device=device, dtype=dtype, ), - fl.Slicing(dim=-1, start=1, length=num_mask_tokens), + fl.Slicing(dim=-1, start=1, end=num_mask_tokens + 1), )