From f912f39b50f87e50a9d99346f5c1b6e644653262 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Thu, 26 Oct 2023 17:19:15 -1000 Subject: [PATCH 01/28] correct checkpoint in kandinsky2.2 doc page (#5550) update checkpoint Co-authored-by: yiyixuxu --- docs/source/en/api/pipelines/kandinsky_v22.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/kandinsky_v22.md b/docs/source/en/api/pipelines/kandinsky_v22.md index 4967ccc0a5cd..44c10fd07789 100644 --- a/docs/source/en/api/pipelines/kandinsky_v22.md +++ b/docs/source/en/api/pipelines/kandinsky_v22.md @@ -237,7 +237,7 @@ to speed-up the optimization. This can be done by simply running: from diffusers import DiffusionPipeline import torch -t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) +t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16) t2i_pipe.enable_xformers_memory_efficient_attention() ``` From 798591346d7256c509c589ffcdc3488c8f2d2e07 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Fri, 27 Oct 2023 21:29:11 +0530 Subject: [PATCH 02/28] [Core] fix FreeU disable method (#5552) * disable freeu debug * debug * potentially fix. * finish * manually remove the spaces * remove tab --- src/diffusers/models/unet_2d_condition.py | 2 +- src/diffusers/models/unet_3d_condition.py | 2 +- .../pipelines/versatile_diffusion/modeling_text_unet.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index fdd0de0a30ff..f248b243f376 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -791,7 +791,7 @@ def disable_freeu(self): freeu_keys = {"s1", "s2", "b1", "b2"} for i, upsample_block in enumerate(self.up_blocks): for k in freeu_keys: - if hasattr(upsample_block, k) or getattr(upsample_block, k) is not None: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: setattr(upsample_block, k, None) def forward( diff --git a/src/diffusers/models/unet_3d_condition.py b/src/diffusers/models/unet_3d_condition.py index 2ab1d4060e17..7356fb577584 100644 --- a/src/diffusers/models/unet_3d_condition.py +++ b/src/diffusers/models/unet_3d_condition.py @@ -494,7 +494,7 @@ def disable_freeu(self): freeu_keys = {"s1", "s2", "b1", "b2"} for i, upsample_block in enumerate(self.up_blocks): for k in freeu_keys: - if hasattr(upsample_block, k) or getattr(upsample_block, k) is not None: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: setattr(upsample_block, k, None) def forward( diff --git a/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py b/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py index 9066c47c56c6..32147ffa455b 100644 --- a/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py +++ b/src/diffusers/pipelines/versatile_diffusion/modeling_text_unet.py @@ -1001,7 +1001,7 @@ def disable_freeu(self): freeu_keys = {"s1", "s2", "b1", "b2"} for i, upsample_block in enumerate(self.up_blocks): for k in freeu_keys: - if hasattr(upsample_block, k) or getattr(upsample_block, k) is not None: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: setattr(upsample_block, k, None) def forward( From 595ba6f786510aaea77219ba4a2647255a41ede1 Mon Sep 17 00:00:00 2001 From: Steven Liu <59462357+stevhliu@users.noreply.github.com> Date: Fri, 27 Oct 2023 09:48:41 -0700 Subject: [PATCH 03/28] [docs] Internal classes API (#5513) * internal classes api * add internal class overview * fix toctree --- docs/source/en/_toctree.yml | 32 +++++++++++-------- .../en/api/internal_classes_overview.md | 3 ++ 2 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 docs/source/en/api/internal_classes_overview.md diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index cef8f474c00e..f2adb148cc28 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -164,24 +164,14 @@ title: Conceptual Guides - sections: - sections: - - local: api/activations - title: Custom activation functions - - local: api/normalization - title: Custom normalization layers - - local: api/attnprocessor - title: Attention Processor - - local: api/logging - title: Logging - local: api/configuration title: Configuration - - local: api/outputs - title: Outputs - local: api/loaders title: Loaders - - local: api/utilities - title: Utilities - - local: api/image_processor - title: VAE Image Processor + - local: api/logging + title: Logging + - local: api/outputs + title: Outputs title: Main Classes - sections: - local: api/models/overview @@ -389,4 +379,18 @@ - local: api/schedulers/vq_diffusion title: VQDiffusionScheduler title: Schedulers + - sections: + - local: api/internal_classes_overview + title: Overview + - local: api/attnprocessor + title: Attention Processor + - local: api/activations + title: Custom activation functions + - local: api/normalization + title: Custom normalization layers + - local: api/utilities + title: Utilities + - local: api/image_processor + title: VAE Image Processor + title: Internal classes title: API diff --git a/docs/source/en/api/internal_classes_overview.md b/docs/source/en/api/internal_classes_overview.md new file mode 100644 index 000000000000..421a22d5ceb5 --- /dev/null +++ b/docs/source/en/api/internal_classes_overview.md @@ -0,0 +1,3 @@ +# Overview + +The APIs in this section are more experimental and prone to breaking changes. Most of them are used internally for development, but they may also be useful to you if you're interested in building a diffusion model with some custom parts or if you're interested in some of our helper utilities for working with 🤗 Diffusers. From e140c0562ed87ec37273796d3e44d2f3aae4d9c2 Mon Sep 17 00:00:00 2001 From: jiaqiw09 <60021713+jiaqiw09@users.noreply.github.com> Date: Fri, 27 Oct 2023 12:19:14 -0500 Subject: [PATCH 04/28] fix error reported 'find_unused_parameters' running in mutiple GPUs (#5355) * fix error reported 'find_unused_parameters' running in mutiple GPUs or NPUs * fix code check of importing module by its alphabetic order --------- Co-authored-by: jiaqiw Co-authored-by: Dhruv Nair --- examples/dreambooth/train_dreambooth_lora_sdxl.py | 5 +++-- examples/text_to_image/train_text_to_image_lora_sdxl.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/dreambooth/train_dreambooth_lora_sdxl.py b/examples/dreambooth/train_dreambooth_lora_sdxl.py index d7df6d4ef526..b729f7e1896d 100644 --- a/examples/dreambooth/train_dreambooth_lora_sdxl.py +++ b/examples/dreambooth/train_dreambooth_lora_sdxl.py @@ -31,7 +31,7 @@ import transformers from accelerate import Accelerator from accelerate.logging import get_logger -from accelerate.utils import ProjectConfiguration, set_seed +from accelerate.utils import DistributedDataParallelKwargs, ProjectConfiguration, set_seed from huggingface_hub import create_repo, upload_folder from packaging import version from PIL import Image @@ -579,12 +579,13 @@ def main(args): logging_dir = Path(args.output_dir, args.logging_dir) accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) - + kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) accelerator = Accelerator( gradient_accumulation_steps=args.gradient_accumulation_steps, mixed_precision=args.mixed_precision, log_with=args.report_to, project_config=accelerator_project_config, + kwargs_handlers=[kwargs], ) if args.report_to == "wandb": diff --git a/examples/text_to_image/train_text_to_image_lora_sdxl.py b/examples/text_to_image/train_text_to_image_lora_sdxl.py index 35de6eedcabd..74fc01aee3e6 100644 --- a/examples/text_to_image/train_text_to_image_lora_sdxl.py +++ b/examples/text_to_image/train_text_to_image_lora_sdxl.py @@ -33,7 +33,7 @@ import transformers from accelerate import Accelerator from accelerate.logging import get_logger -from accelerate.utils import ProjectConfiguration, set_seed +from accelerate.utils import DistributedDataParallelKwargs, ProjectConfiguration, set_seed from datasets import load_dataset from huggingface_hub import create_repo, upload_folder from packaging import version @@ -491,12 +491,13 @@ def main(args): logging_dir = Path(args.output_dir, args.logging_dir) accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) - + kwargs = DistributedDataParallelKwargs(find_unused_parameters=True) accelerator = Accelerator( gradient_accumulation_steps=args.gradient_accumulation_steps, mixed_precision=args.mixed_precision, log_with=args.report_to, project_config=accelerator_project_config, + kwargs_handlers=[kwargs], ) if args.report_to == "wandb": From 9135e54e768a59ddcf8ad18818d2ffe69ea3a32a Mon Sep 17 00:00:00 2001 From: Gabriel de Souza Date: Fri, 27 Oct 2023 14:51:35 -0300 Subject: [PATCH 05/28] docs: initial pt translation (#5549) * docs: initial pt translation * docs: add pt build to github workflow and fix some missing translations --- .github/workflows/build_documentation.yml | 2 +- .github/workflows/build_pr_documentation.yml | 2 +- docs/source/pt/_toctree.yml | 8 + docs/source/pt/index.md | 48 +++ docs/source/pt/installation.md | 156 +++++++++ docs/source/pt/quicktour.md | 314 +++++++++++++++++++ 6 files changed, 528 insertions(+), 2 deletions(-) create mode 100644 docs/source/pt/_toctree.yml create mode 100644 docs/source/pt/index.md create mode 100644 docs/source/pt/installation.md create mode 100644 docs/source/pt/quicktour.md diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 67229d634c91..05ea1b2122de 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -16,7 +16,7 @@ jobs: install_libgl1: true package: diffusers notebook_folder: diffusers_doc - languages: en ko zh ja + languages: en ko zh ja pt secrets: token: ${{ secrets.HUGGINGFACE_PUSH }} diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index f5b666ee27ff..33f09b309d49 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -15,4 +15,4 @@ jobs: pr_number: ${{ github.event.number }} install_libgl1: true package: diffusers - languages: en ko zh ja + languages: en ko zh ja pt diff --git a/docs/source/pt/_toctree.yml b/docs/source/pt/_toctree.yml new file mode 100644 index 000000000000..c34297a4743f --- /dev/null +++ b/docs/source/pt/_toctree.yml @@ -0,0 +1,8 @@ +- sections: + - local: index + title: 🧨 Diffusers + - local: quicktour + title: Tour rápido + - local: installation + title: Instalação + title: Primeiros passos diff --git a/docs/source/pt/index.md b/docs/source/pt/index.md new file mode 100644 index 000000000000..b524fa6449e9 --- /dev/null +++ b/docs/source/pt/index.md @@ -0,0 +1,48 @@ + + +

+
+ +
+

+ +# Diffusers + +🤗 Diffusers é uma biblioteca de modelos de difusão de última geração para geração de imagens, áudio e até mesmo estruturas 3D de moléculas. Se você está procurando uma solução de geração simples ou queira treinar seu próprio modelo de difusão, 🤗 Diffusers é uma modular caixa de ferramentas que suporta ambos. Nossa biblioteca é desenhada com foco em [usabilidade em vez de desempenho](conceptual/philosophy#usability-over-performance), [simples em vez de fácil](conceptual/philosophy#simple-over-easy) e [customizável em vez de abstrações](conceptual/philosophy#tweakable-contributorfriendly-over-abstraction). + +A Biblioteca tem três componentes principais: + +- Pipelines de última geração para a geração em poucas linhas de código. Têm muitos pipelines no 🤗 Diffusers, veja a tabela no pipeline [Visão geral](api/pipelines/overview) para uma lista completa de pipelines disponíveis e as tarefas que eles resolvem. +- Intercambiáveis [agendadores de ruído](api/schedulers/overview) para balancear as compensações entre velocidade e qualidade de geração. +- [Modelos](api/models) pré-treinados que podem ser usados como se fossem blocos de construção, e combinados com agendadores, para criar seu próprio sistema de difusão de ponta a ponta. + +
+
+
Tutoriais
+

Aprenda as competências fundamentais que precisa para iniciar a gerar saídas, construa seu próprio sistema de difusão, e treine um modelo de difusão. Nós recomendamos começar por aqui se você está utilizando o 🤗 Diffusers pela primeira vez!

+ +
Guias de utilização
+

Guias práticos para ajudar você carregar pipelines, modelos, e agendadores. Você também aprenderá como usar os pipelines para tarefas específicas, controlar como as saídas são geradas, otimizar a velocidade de geração, e outras técnicas diferentes de treinamento.

+ +
Guias conceituais
+

Compreenda porque a biblioteca foi desenhada da forma que ela é, e aprenda mais sobre as diretrizes éticas e implementações de segurança para o uso da biblioteca.

+ +
Referência
+

Descrições técnicas de como funcionam as classes e métodos do 🤗 Diffusers

+ +
+
diff --git a/docs/source/pt/installation.md b/docs/source/pt/installation.md new file mode 100644 index 000000000000..52ea243ee034 --- /dev/null +++ b/docs/source/pt/installation.md @@ -0,0 +1,156 @@ + + +# Instalação + +🤗 Diffusers é testado no Python 3.8+, PyTorch 1.7.0+, e Flax. Siga as instruções de instalação abaixo para a biblioteca de deep learning que você está utilizando: + +- [PyTorch](https://pytorch.org/get-started/locally/) instruções de instalação +- [Flax](https://flax.readthedocs.io/en/latest/) instruções de instalação + +## Instalação com pip + +Recomenda-se instalar 🤗 Diffusers em um [ambiente virtual](https://docs.python.org/3/library/venv.html). +Se você não está familiarizado com ambiente virtuals, veja o [guia](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/). +Um ambiente virtual deixa mais fácil gerenciar diferentes projetos e evitar problemas de compatibilidade entre dependências. + +Comece criando um ambiente virtual no diretório do projeto: + +```bash +python -m venv .env +``` + +Ative o ambiente virtual: + +```bash +source .env/bin/activate +``` + +Recomenda-se a instalação do 🤗 Transformers porque 🤗 Diffusers depende de seus modelos: + + + +```bash +pip install diffusers["torch"] transformers +``` + + +```bash +pip install diffusers["flax"] transformers +``` + + + +## Instalação a partir do código fonte + +Antes da instalação do 🤗 Diffusers a partir do código fonte, certifique-se de ter o PyTorch e o 🤗 Accelerate instalados. + +Para instalar o 🤗 Accelerate: + +```bash +pip install accelerate +``` + +então instale o 🤗 Diffusers do código fonte: + +```bash +pip install git+https://github.com/huggingface/diffusers +``` + +Esse comando instala a última versão em desenvolvimento `main` em vez da última versão estável `stable`. +A versão `main` é útil para se manter atualizado com os últimos desenvolvimentos. +Por exemplo, se um bug foi corrigido desde o último lançamento estável, mas um novo lançamento ainda não foi lançado. +No entanto, isso significa que a versão `main` pode não ser sempre estável. +Nós nos esforçamos para manter a versão `main` operacional, e a maioria dos problemas geralmente são resolvidos em algumas horas ou um dia. +Se você encontrar um problema, por favor abra uma [Issue](https://github.com/huggingface/diffusers/issues/new/choose), assim conseguimos arrumar o quanto antes! + +## Instalação editável + +Você precisará de uma instalação editável se você: + +- Usar a versão `main` do código fonte. +- Contribuir para o 🤗 Diffusers e precisa testar mudanças no código. + +Clone o repositório e instale o 🤗 Diffusers com os seguintes comandos: + +```bash +git clone https://github.com/huggingface/diffusers.git +cd diffusers +``` + + + +```bash +pip install -e ".[torch]" +``` + + +```bash +pip install -e ".[flax]" +``` + + + +Esses comandos irá linkar a pasta que você clonou o repositório e os caminhos das suas bibliotecas Python. +Python então irá procurar dentro da pasta que você clonou além dos caminhos normais das bibliotecas. +Por exemplo, se o pacote python for tipicamente instalado no `~/anaconda3/envs/main/lib/python3.8/site-packages/`, o Python também irá procurar na pasta `~/diffusers/` que você clonou. + + + +Você deve deixar a pasta `diffusers` se você quiser continuar usando a biblioteca. + + + +Agora você pode facilmente atualizar seu clone para a última versão do 🤗 Diffusers com o seguinte comando: + +```bash +cd ~/diffusers/ +git pull +``` + +Seu ambiente Python vai encontrar a versão `main` do 🤗 Diffusers na próxima execução. + +## Cache + +Os pesos e os arquivos dos modelos são baixados do Hub para o cache que geralmente é o seu diretório home. Você pode mudar a localização do cache especificando as variáveis de ambiente `HF_HOME` ou `HUGGINFACE_HUB_CACHE` ou configurando o parâmetro `cache_dir` em métodos como [`~DiffusionPipeline.from_pretrained`]. + +Aquivos em cache permitem que você rode 🤗 Diffusers offline. Para prevenir que o 🤗 Diffusers se conecte à internet, defina a variável de ambiente `HF_HUB_OFFLINE` para `True` e o 🤗 Diffusers irá apenas carregar arquivos previamente baixados em cache. + +```shell +export HF_HUB_OFFLINE=True +``` + +Para mais detalhes de como gerenciar e limpar o cache, olhe o guia de [caching](https://huggingface.co/docs/huggingface_hub/guides/manage-cache). + +## Telemetria + +Nossa biblioteca coleta informações de telemetria durante as requisições [`~DiffusionPipeline.from_pretrained`]. +O dado coletado inclui a versão do 🤗 Diffusers e PyTorch/Flax, o modelo ou classe de pipeline requisitado, +e o caminho para um checkpoint pré-treinado se ele estiver hospedado no Hugging Face Hub. +Esse dado de uso nos ajuda a debugar problemas e priorizar novas funcionalidades. +Telemetria é enviada apenas quando é carregado modelos e pipelines do Hub, +e não é coletado se você estiver carregando arquivos locais. + +Nos entendemos que nem todo mundo quer compartilhar informações adicionais, e nós respeitamos sua privacidade. +Você pode desabilitar a coleta de telemetria definindo a variável de ambiente `DISABLE_TELEMETRY` do seu terminal: + +No Linux/MacOS: + +```bash +export DISABLE_TELEMETRY=YES +``` + +No Windows: + +```bash +set DISABLE_TELEMETRY=YES +``` diff --git a/docs/source/pt/quicktour.md b/docs/source/pt/quicktour.md new file mode 100644 index 000000000000..1fae0e45e39f --- /dev/null +++ b/docs/source/pt/quicktour.md @@ -0,0 +1,314 @@ + + +[[open-in-colab]] + +# Tour rápido + +Modelos de difusão são treinados para remover o ruído Gaussiano aleatório passo a passo para gerar uma amostra de interesse, como uma imagem ou áudio. Isso despertou um tremendo interesse em IA generativa, e você provavelmente já viu exemplos de imagens geradas por difusão na internet. 🧨 Diffusers é uma biblioteca que visa tornar os modelos de difusão amplamente acessíveis a todos. + +Seja você um desenvolvedor ou um usuário, esse tour rápido irá introduzir você ao 🧨 Diffusers e ajudar você a começar a gerar rapidamente! Há três componentes principais da biblioteca para conhecer: + +- O [`DiffusionPipeline`] é uma classe de alto nível de ponta a ponta desenhada para gerar rapidamente amostras de modelos de difusão pré-treinados para inferência. +- [Modelos](./api/models) pré-treinados populares e módulos que podem ser usados como blocos de construção para criar sistemas de difusão. +- Vários [Agendadores](./api/schedulers/overview) diferentes - algoritmos que controlam como o ruído é adicionado para treinamento, e como gerar imagens sem o ruído durante a inferência. + +Esse tour rápido mostrará como usar o [`DiffusionPipeline`] para inferência, e então mostrará como combinar um modelo e um agendador para replicar o que está acontecendo dentro do [`DiffusionPipeline`]. + + + +Esse tour rápido é uma versão simplificada da introdução 🧨 Diffusers [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/diffusers_intro.ipynb) para ajudar você a começar rápido. Se você quer aprender mais sobre o objetivo do 🧨 Diffusers, filosofia de design, e detalhes adicionais sobre a API principal, veja o notebook! + + + +Antes de começar, certifique-se de ter todas as bibliotecas necessárias instaladas: + +```py +# uncomment to install the necessary libraries in Colab +#!pip install --upgrade diffusers accelerate transformers +``` + +- [🤗 Accelerate](https://huggingface.co/docs/accelerate/index) acelera o carregamento do modelo para geração e treinamento. +- [🤗 Transformers](https://huggingface.co/docs/transformers/index) é necessário para executar os modelos mais populares de difusão, como o [Stable Diffusion](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/overview). + +## DiffusionPipeline + +O [`DiffusionPipeline`] é a forma mais fácil de usar um sistema de difusão pré-treinado para geração. É um sistema de ponta a ponta contendo o modelo e o agendador. Você pode usar o [`DiffusionPipeline`] pronto para muitas tarefas. Dê uma olhada na tabela abaixo para algumas tarefas suportadas, e para uma lista completa de tarefas suportadas, veja a tabela [Resumo do 🧨 Diffusers](./api/pipelines/overview#diffusers-summary). + +| **Tarefa** | **Descrição** | **Pipeline** | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| Unconditional Image Generation | gera uma imagem a partir do ruído Gaussiano | [unconditional_image_generation](./using-diffusers/unconditional_image_generation) | +| Text-Guided Image Generation | gera uma imagem a partir de um prompt de texto | [conditional_image_generation](./using-diffusers/conditional_image_generation) | +| Text-Guided Image-to-Image Translation | adapta uma imagem guiada por um prompt de texto | [img2img](./using-diffusers/img2img) | +| Text-Guided Image-Inpainting | preenche a parte da máscara da imagem, dado a imagem, a máscara e o prompt de texto | [inpaint](./using-diffusers/inpaint) | +| Text-Guided Depth-to-Image Translation | adapta as partes de uma imagem guiada por um prompt de texto enquanto preserva a estrutura por estimativa de profundidade | [depth2img](./using-diffusers/depth2img) | + +Comece criando uma instância do [`DiffusionPipeline`] e especifique qual checkpoint do pipeline você gostaria de baixar. +Você pode usar o [`DiffusionPipeline`] para qualquer [checkpoint](https://huggingface.co/models?library=diffusers&sort=downloads) armazenado no Hugging Face Hub. +Nesse quicktour, você carregará o checkpoint [`stable-diffusion-v1-5`](https://huggingface.co/runwayml/stable-diffusion-v1-5) para geração de texto para imagem. + + + +Para os modelos de [Stable Diffusion](https://huggingface.co/CompVis/stable-diffusion), por favor leia cuidadosamente a [licença](https://huggingface.co/spaces/CompVis/stable-diffusion-license) primeiro antes de rodar o modelo. 🧨 Diffusers implementa uma verificação de segurança: [`safety_checker`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/safety_checker.py) para prevenir conteúdo ofensivo ou nocivo, mas as capacidades de geração de imagem aprimorada do modelo podem ainda produzir conteúdo potencialmente nocivo. + + + +Para carregar o modelo com o método [`~DiffusionPipeline.from_pretrained`]: + +```python +>>> from diffusers import DiffusionPipeline + +>>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_safetensors=True) +``` + +O [`DiffusionPipeline`] baixa e armazena em cache todos os componentes de modelagem, tokenização, e agendamento. Você verá que o pipeline do Stable Diffusion é composto pelo [`UNet2DConditionModel`] e [`PNDMScheduler`] entre outras coisas: + +```py +>>> pipeline +StableDiffusionPipeline { + "_class_name": "StableDiffusionPipeline", + "_diffusers_version": "0.13.1", + ..., + "scheduler": [ + "diffusers", + "PNDMScheduler" + ], + ..., + "unet": [ + "diffusers", + "UNet2DConditionModel" + ], + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} +``` + +Nós fortemente recomendamos rodar o pipeline em uma placa de vídeo, pois o modelo consiste em aproximadamente 1.4 bilhões de parâmetros. +Você pode mover o objeto gerador para uma placa de vídeo, assim como você faria no PyTorch: + +```python +>>> pipeline.to("cuda") +``` + +Agora você pode passar o prompt de texto para o `pipeline` para gerar uma imagem, e então acessar a imagem sem ruído. Por padrão, a saída da imagem é embrulhada em um objeto [`PIL.Image`](https://pillow.readthedocs.io/en/stable/reference/Image.html?highlight=image#the-image-class). + +```python +>>> image = pipeline("An image of a squirrel in Picasso style").images[0] +>>> image +``` + +
+ +
+ +Salve a imagem chamando o `save`: + +```python +>>> image.save("image_of_squirrel_painting.png") +``` + +### Pipeline local + +Você também pode utilizar o pipeline localmente. A única diferença é que você precisa baixar os pesos primeiro: + +```bash +!git lfs install +!git clone https://huggingface.co/runwayml/stable-diffusion-v1-5 +``` + +Assim carregue os pesos salvos no pipeline: + +```python +>>> pipeline = DiffusionPipeline.from_pretrained("./stable-diffusion-v1-5", use_safetensors=True) +``` + +Agora você pode rodar o pipeline como você faria na seção acima. + +### Troca dos agendadores + +Agendadores diferentes tem diferentes velocidades de retirar o ruído e compensações de qualidade. A melhor forma de descobrir qual funciona melhor para você é testar eles! Uma das principais características do 🧨 Diffusers é permitir que você troque facilmente entre agendadores. Por exemplo, para substituir o [`PNDMScheduler`] padrão com o [`EulerDiscreteScheduler`], carregue ele com o método [`~diffusers.ConfigMixin.from_config`]: + +```py +>>> from diffusers import EulerDiscreteScheduler + +>>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_safetensors=True) +>>> pipeline.scheduler = EulerDiscreteScheduler.from_config(pipeline.scheduler.config) +``` + +Tente gerar uma imagem com o novo agendador e veja se você nota alguma diferença! + +Na próxima seção, você irá dar uma olhada mais de perto nos componentes - o modelo e o agendador - que compõe o [`DiffusionPipeline`] e aprender como usar esses componentes para gerar uma imagem de um gato. + +## Modelos + +A maioria dos modelos recebe uma amostra de ruído, e em cada _timestep_ ele prevê o _noise residual_ (outros modelos aprendem a prever a amostra anterior diretamente ou a velocidade ou [`v-prediction`](https://github.com/huggingface/diffusers/blob/5e5ce13e2f89ac45a0066cb3f369462a3cf1d9ef/src/diffusers/schedulers/scheduling_ddim.py#L110)), a diferença entre uma imagem menos com ruído e a imagem de entrada. Você pode misturar e combinar modelos para criar outros sistemas de difusão. + +Modelos são inicializados com o método [`~ModelMixin.from_pretrained`] que também armazena em cache localmente os pesos do modelo para que seja mais rápido na próxima vez que você carregar o modelo. Para o tour rápido, você irá carregar o [`UNet2DModel`], um modelo básico de geração de imagem incondicional com um checkpoint treinado em imagens de gato: + +```py +>>> from diffusers import UNet2DModel + +>>> repo_id = "google/ddpm-cat-256" +>>> model = UNet2DModel.from_pretrained(repo_id, use_safetensors=True) +``` + +Para acessar os parâmetros do modelo, chame `model.config`: + +```py +>>> model.config +``` + +A configuração do modelo é um dicionário 🧊 congelado 🧊, o que significa que esses parâmetros não podem ser mudados depois que o modelo é criado. Isso é intencional e garante que os parâmetros usados para definir a arquitetura do modelo no início permaneçam os mesmos, enquanto outros parâmetros ainda podem ser ajustados durante a geração. + +Um dos parâmetros mais importantes são: + +- `sample_size`: a dimensão da altura e largura da amostra de entrada. +- `in_channels`: o número de canais de entrada da amostra de entrada. +- `down_block_types` e `up_block_types`: o tipo de blocos de downsampling e upsampling usados para criar a arquitetura UNet. +- `block_out_channels`: o número de canais de saída dos blocos de downsampling; também utilizado como uma order reversa do número de canais de entrada dos blocos de upsampling. +- `layers_per_block`: o número de blocks ResNet presentes em cada block UNet. + +Para usar o modelo para geração, crie a forma da imagem com ruído Gaussiano aleatório. Deve ter um eixo `batch` porque o modelo pode receber múltiplos ruídos aleatórios, um eixo `channel` correspondente ao número de canais de entrada, e um eixo `sample_size` para a altura e largura da imagem: + +```py +>>> import torch + +>>> torch.manual_seed(0) + +>>> noisy_sample = torch.randn(1, model.config.in_channels, model.config.sample_size, model.config.sample_size) +>>> noisy_sample.shape +torch.Size([1, 3, 256, 256]) +``` + +Para geração, passe a imagem com ruído para o modelo e um `timestep`. O `timestep` indica o quão ruidosa a imagem de entrada é, com mais ruído no início e menos no final. Isso ajuda o modelo a determinar sua posição no processo de difusão, se está mais perto do início ou do final. Use o método `sample` para obter a saída do modelo: + +```py +>>> with torch.no_grad(): +... noisy_residual = model(sample=noisy_sample, timestep=2).sample +``` + +Para geração de exemplos reais, você precisará de um agendador para guiar o processo de retirada do ruído. Na próxima seção, você irá aprender como acoplar um modelo com um agendador. + +## Agendadores + +Agendadores gerenciam a retirada do ruído de uma amostra ruidosa para uma amostra menos ruidosa dado a saída do modelo - nesse caso, é o `noisy_residual`. + + + +🧨 Diffusers é uma caixa de ferramentas para construir sistemas de difusão. Enquanto o [`DiffusionPipeline`] é uma forma conveniente de começar com um sistema de difusão pré-construído, você também pode escolher seus próprios modelos e agendadores separadamente para construir um sistema de difusão personalizado. + + + +Para o tour rápido, você irá instanciar o [`DDPMScheduler`] com o método [`~diffusers.ConfigMixin.from_config`]: + +```py +>>> from diffusers import DDPMScheduler + +>>> scheduler = DDPMScheduler.from_config(repo_id) +>>> scheduler +DDPMScheduler { + "_class_name": "DDPMScheduler", + "_diffusers_version": "0.13.1", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": true, + "clip_sample_range": 1.0, + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "trained_betas": null, + "variance_type": "fixed_small" +} +``` + + + +💡 Perceba como o agendador é instanciado de uma configuração. Diferentemente de um modelo, um agendador não tem pesos treináveis e é livre de parâmetros! + + + +Um dos parâmetros mais importante são: + +- `num_train_timesteps`: o tamanho do processo de retirar ruído ou em outras palavras, o número de _timesteps_ necessários para o processo de ruídos Gausianos aleatórios dentro de uma amostra de dados. +- `beta_schedule`: o tipo de agendados de ruído para o uso de geração e treinamento. +- `beta_start` e `beta_end`: para começar e terminar os valores de ruído para o agendador de ruído. + +Para predizer uma imagem com um pouco menos de ruído, passe o seguinte para o método do agendador [`~diffusers.DDPMScheduler.step`]: saída do modelo, `timestep`, e a atual `amostra`. + +```py +>>> less_noisy_sample = scheduler.step(model_output=noisy_residual, timestep=2, sample=noisy_sample).prev_sample +>>> less_noisy_sample.shape +``` + +O `less_noisy_sample` pode ser passado para o próximo `timestep` onde ele ficará ainda com menos ruído! Vamos juntar tudo agora e visualizar o processo inteiro de retirada de ruído. + +Comece, criando a função que faça o pós-processamento e mostre a imagem sem ruído como uma `PIL.Image`: + +```py +>>> import PIL.Image +>>> import numpy as np + + +>>> def display_sample(sample, i): +... image_processed = sample.cpu().permute(0, 2, 3, 1) +... image_processed = (image_processed + 1.0) * 127.5 +... image_processed = image_processed.numpy().astype(np.uint8) + +... image_pil = PIL.Image.fromarray(image_processed[0]) +... display(f"Image at step {i}") +... display(image_pil) +``` + +Para acelerar o processo de retirada de ruído, mova a entrada e o modelo para uma GPU: + +```py +>>> model.to("cuda") +>>> noisy_sample = noisy_sample.to("cuda") +``` + +Agora, crie um loop de retirada de ruído que prediz o residual da amostra menos ruidosa, e computa a amostra menos ruidosa com o agendador: + +```py +>>> import tqdm + +>>> sample = noisy_sample + +>>> for i, t in enumerate(tqdm.tqdm(scheduler.timesteps)): +... # 1. predict noise residual +... with torch.no_grad(): +... residual = model(sample, t).sample + +... # 2. compute less noisy image and set x_t -> x_t-1 +... sample = scheduler.step(residual, t, sample).prev_sample + +... # 3. optionally look at image +... if (i + 1) % 50 == 0: +... display_sample(sample, i + 1) +``` + +Sente-se e assista o gato ser gerado do nada além de ruído! 😻 + +
+ +
+ +## Próximos passos + +Esperamos que você tenha gerado algumas imagens legais com o 🧨 Diffusers neste tour rápido! Para suas próximas etapas, você pode + +- Treine ou faça a configuração fina de um modelo para gerar suas próprias imagens no tutorial de [treinamento](./tutorials/basic_training). +- Veja exemplos oficiais e da comunidade de [scripts de treinamento ou configuração fina](https://github.com/huggingface/diffusers/tree/main/examples#-diffusers-examples) para os mais variados casos de uso. +- Aprenda sobre como carregar, acessar, mudar e comparar agendadores no guia [Usando diferentes agendadores](./using-diffusers/schedulers). +- Explore engenharia de prompt, otimizações de velocidade e memória, e dicas e truques para gerar imagens de maior qualidade com o guia [Stable Diffusion](./stable_diffusion). +- Se aprofunde em acelerar 🧨 Diffusers com guias sobre [PyTorch otimizado em uma GPU](./optimization/fp16), e guias de inferência para rodar [Stable Diffusion em Apple Silicon (M1/M2)](./optimization/mps) e [ONNX Runtime](./optimization/onnx). From 3ec828d6dd8ee7731fb9e51c2dd061fffebadd87 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Mon, 30 Oct 2023 14:25:31 +0100 Subject: [PATCH 06/28] Fix moved _expand_mask function (#5581) * finish * finish --- .../blip_diffusion/modeling_ctx_clip.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py b/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py index 53d57188743d..19f62e789e2d 100644 --- a/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +++ b/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py @@ -19,10 +19,21 @@ from transformers import CLIPPreTrainedModel from transformers.modeling_outputs import BaseModelOutputWithPooling from transformers.models.clip.configuration_clip import CLIPTextConfig -from transformers.models.clip.modeling_clip import ( - CLIPEncoder, - _expand_mask, -) +from transformers.models.clip.modeling_clip import CLIPEncoder + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) # This is a modified version of the CLIPTextModel from transformers.models.clip.modeling_clip From 8f3100db9fbbc0409ad10aa2475f6dd25190f592 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 30 Oct 2023 14:27:00 +0100 Subject: [PATCH 07/28] [`PEFT` / `Tests`] Add peft slow tests on push (#5419) * add peft slow tests workflow * Update .github/workflows/push_tests.yml --------- Co-authored-by: Sayak Paul --- .github/workflows/push_tests.yml | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 5fadd095be35..dbf48ca8f158 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -156,6 +156,56 @@ jobs: name: torch_cuda_test_reports path: reports + peft_cuda_tests: + name: PEFT CUDA Tests + runs-on: docker-gpu + container: + image: diffusers/diffusers-pytorch-cuda + options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ --gpus 0 + defaults: + run: + shell: bash + steps: + - name: Checkout diffusers + uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Install dependencies + run: | + apt-get update && apt-get install libsndfile1-dev libgl1 -y + python -m pip install -e .[quality,test] + python -m pip install git+https://github.com/huggingface/accelerate.git + python -m pip install git+https://github.com/huggingface/peft.git + + - name: Environment + run: | + python utils/print_env.py + + - name: Run slow PEFT CUDA tests + env: + HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms + CUBLAS_WORKSPACE_CONFIG: :16:8 + run: | + python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \ + -s -v -k "not Flax and not Onnx" \ + --make-reports=tests_peft_cuda \ + tests/lora/ + + - name: Failure short reports + if: ${{ failure() }} + run: | + cat reports/tests_peft_cuda_stats.txt + cat reports/tests_peft_cuda_failures_short.txt + + - name: Test suite reports artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: torch_peft_test_reports + path: reports + flax_tpu_tests: name: Flax TPU Tests runs-on: docker-tpu From 5b087e82d1b7cc65860f69364f14a73f398c6de0 Mon Sep 17 00:00:00 2001 From: "Thuan H. Nguyen" <32274287+thuanz123@users.noreply.github.com> Date: Mon, 30 Oct 2023 21:21:40 +0700 Subject: [PATCH 08/28] Add realfill (#5456) * Add realfill * Move realfill folder * Fix some format issues --- examples/research_projects/realfill/README.md | 118 +++ examples/research_projects/realfill/infer.py | 91 ++ .../realfill/requirements.txt | 9 + .../realfill/train_realfill.py | 977 ++++++++++++++++++ 4 files changed, 1195 insertions(+) create mode 100644 examples/research_projects/realfill/README.md create mode 100644 examples/research_projects/realfill/infer.py create mode 100644 examples/research_projects/realfill/requirements.txt create mode 100644 examples/research_projects/realfill/train_realfill.py diff --git a/examples/research_projects/realfill/README.md b/examples/research_projects/realfill/README.md new file mode 100644 index 000000000000..b70f425368e5 --- /dev/null +++ b/examples/research_projects/realfill/README.md @@ -0,0 +1,118 @@ +# RealFill + +[RealFill](https://arxiv.org/abs/2309.16668) is a method to personalize text2image inpainting models like stable diffusion inpainting given just a few(1~5) images of a scene. +The `train_realfill.py` script shows how to implement the training procedure for stable diffusion inpainting. + + +## Running locally with PyTorch + +### Installing the dependencies + +Before running the scripts, make sure to install the library's training dependencies: + +cd to the realfill folder and run +```bash +cd realfill +pip install -r requirements.txt +``` + +And initialize an [🤗Accelerate](https://github.com/huggingface/accelerate/) environment with: + +```bash +accelerate config +``` + +Or for a default accelerate configuration without answering questions about your environment + +```bash +accelerate config default +``` + +Or if your environment doesn't support an interactive shell e.g. a notebook + +```python +from accelerate.utils import write_basic_config +write_basic_config() +``` + +When running `accelerate config`, if we specify torch compile mode to True there can be dramatic speedups. + +### Toy example + +Now let's fill the real. For this example, we will use some images of the flower girl example from the paper. + +We already provide some images for testing in [this link](https://github.com/thuanz123/realfill/tree/main/data/flowerwoman) + +You only have to launch the training using: + +```bash +export MODEL_NAME="stabilityai/stable-diffusion-2-inpainting" +export TRAIN_DIR="data/flowerwoman" +export OUTPUT_DIR="flowerwoman-model" + +accelerate launch train_realfill.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$TRAIN_DIR \ + --output_dir=$OUTPUT_DIR \ + --resolution=512 \ + --train_batch_size=16 \ + --gradient_accumulation_steps=1 \ + --unet_learning_rate=2e-4 \ + --text_encoder_learning_rate=4e-5 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=100 \ + --max_train_steps=2000 \ + --lora_rank=8 \ + --lora_dropout=0.1 \ + --lora_alpha=16 \ +``` + +### Training on a low-memory GPU: + +It is possible to run realfill on a low-memory GPU by using the following optimizations: +- [gradient checkpointing and the 8-bit optimizer](#training-with-gradient-checkpointing-and-8-bit-optimizers) +- [xformers](#training-with-xformers) +- [setting grads to none](#set-grads-to-none) + +```bash +export MODEL_NAME="stabilityai/stable-diffusion-2-inpainting" +export TRAIN_DIR="data/flowerwoman" +export OUTPUT_DIR="flowerwoman-model" + +accelerate launch train_realfill.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$TRAIN_DIR \ + --output_dir=$OUTPUT_DIR \ + --resolution=512 \ + --train_batch_size=16 \ + --gradient_accumulation_steps=1 --gradient_checkpointing \ + --use_8bit_adam \ + --enable_xformers_memory_efficient_attention \ + --set_grads_to_none \ + --unet_learning_rate=2e-4 \ + --text_encoder_learning_rate=4e-5 \ + --lr_scheduler="constant" \ + --lr_warmup_steps=100 \ + --max_train_steps=2000 \ + --lora_rank=8 \ + --lora_dropout=0.1 \ + --lora_alpha=16 \ +``` + +### Training with gradient checkpointing and 8-bit optimizers: + +With the help of gradient checkpointing and the 8-bit optimizer from bitsandbytes it's possible to run train realfill on a 16GB GPU. + +To install `bitsandbytes` please refer to this [readme](https://github.com/TimDettmers/bitsandbytes#requirements--installation). + +### Training with xformers: +You can enable memory efficient attention by [installing xFormers](https://github.com/facebookresearch/xformers#installing-xformers) and padding the `--enable_xformers_memory_efficient_attention` argument to the script. + +### Set grads to none + +To save even more memory, pass the `--set_grads_to_none` argument to the script. This will set grads to None instead of zero. However, be aware that it changes certain behaviors, so if you start experiencing any problems, remove this argument. + +More info: https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html + +## Acknowledge +This repo is built upon the code of DreamBooth from diffusers and we thank the developers for their great works and efforts to release source code. Furthermore, a special "thank you" to RealFill's authors for publishing such an amazing work. diff --git a/examples/research_projects/realfill/infer.py b/examples/research_projects/realfill/infer.py new file mode 100644 index 000000000000..3153307c4ad3 --- /dev/null +++ b/examples/research_projects/realfill/infer.py @@ -0,0 +1,91 @@ +import argparse +import os + +import torch +from PIL import Image, ImageFilter +from transformers import CLIPTextModel + +from diffusers import DPMSolverMultistepScheduler, StableDiffusionInpaintPipeline, UNet2DConditionModel + + +parser = argparse.ArgumentParser(description="Inference") +parser.add_argument( + "--model_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", +) +parser.add_argument( + "--validation_image", + type=str, + default=None, + required=True, + help="The directory of the validation image", +) +parser.add_argument( + "--validation_mask", + type=str, + default=None, + required=True, + help="The directory of the validation mask", +) +parser.add_argument( + "--output_dir", + type=str, + default="./test-infer/", + help="The output directory where predictions are saved", +) +parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible inference.") + +args = parser.parse_args() + +if __name__ == "__main__": + os.makedirs(args.output_dir, exist_ok=True) + generator = None + + # create & load model + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-inpainting", torch_dtype=torch.float32, revision=None + ) + + pipe.unet = UNet2DConditionModel.from_pretrained( + args.model_path, + subfolder="unet", + revision=None, + ) + pipe.text_encoder = CLIPTextModel.from_pretrained( + args.model_path, + subfolder="text_encoder", + revision=None, + ) + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe = pipe.to("cuda") + + if args.seed is not None: + generator = torch.Generator(device="cuda").manual_seed(args.seed) + + image = Image.open(args.validation_image) + mask_image = Image.open(args.validation_mask) + + results = pipe( + ["a photo of sks"] * 16, + image=image, + mask_image=mask_image, + num_inference_steps=25, + guidance_scale=5, + generator=generator, + ).images + + erode_kernel = ImageFilter.MaxFilter(3) + mask_image = mask_image.filter(erode_kernel) + + blur_kernel = ImageFilter.BoxBlur(1) + mask_image = mask_image.filter(blur_kernel) + + for idx, result in enumerate(results): + result = Image.composite(result, image, mask_image) + result.save(f"{args.output_dir}/{idx}.png") + + del pipe + torch.cuda.empty_cache() diff --git a/examples/research_projects/realfill/requirements.txt b/examples/research_projects/realfill/requirements.txt new file mode 100644 index 000000000000..bf14291f53a9 --- /dev/null +++ b/examples/research_projects/realfill/requirements.txt @@ -0,0 +1,9 @@ +diffusers==0.20.1 +accelerate==0.23.0 +transformers==4.34.0 +peft==0.5.0 +torch==2.0.1 +torchvision==0.15.2 +ftfy==6.1.1 +tensorboard==2.14.0 +Jinja2==3.1.2 diff --git a/examples/research_projects/realfill/train_realfill.py b/examples/research_projects/realfill/train_realfill.py new file mode 100644 index 000000000000..9d00a21b1a74 --- /dev/null +++ b/examples/research_projects/realfill/train_realfill.py @@ -0,0 +1,977 @@ +import argparse +import copy +import itertools +import logging +import math +import os +import random +import shutil +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.v2 as transforms_v2 +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import set_seed +from huggingface_hub import create_repo, upload_folder +from packaging import version +from peft import LoraConfig, PeftModel, get_peft_model +from PIL import Image +from PIL.ImageOps import exif_transpose +from torch.utils.data import Dataset +from tqdm.auto import tqdm +from transformers import AutoTokenizer, CLIPTextModel + +import diffusers +from diffusers import ( + AutoencoderKL, + DDPMScheduler, + DPMSolverMultistepScheduler, + StableDiffusionInpaintPipeline, + UNet2DConditionModel, +) +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, is_wandb_available +from diffusers.utils.import_utils import is_xformers_available + + +if is_wandb_available(): + import wandb + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.20.1") + +logger = get_logger(__name__) + + +def make_mask(images, resolution, times=30): + mask, times = torch.ones_like(images[0:1, :, :]), np.random.randint(1, times) + min_size, max_size, margin = np.array([0.03, 0.25, 0.01]) * resolution + max_size = min(max_size, resolution - margin * 2) + + for _ in range(times): + width = np.random.randint(int(min_size), int(max_size)) + height = np.random.randint(int(min_size), int(max_size)) + + x_start = np.random.randint(int(margin), resolution - int(margin) - width + 1) + y_start = np.random.randint(int(margin), resolution - int(margin) - height + 1) + mask[:, y_start : y_start + height, x_start : x_start + width] = 0 + + mask = 1 - mask if random.random() < 0.5 else mask + return mask + + +def save_model_card( + repo_id: str, + images=None, + base_model=str, + repo_folder=None, +): + img_str = "" + for i, image in enumerate(images): + image.save(os.path.join(repo_folder, f"image_{i}.png")) + img_str += f"![img_{i}](./image_{i}.png)\n" + + yaml = f""" +--- +license: creativeml-openrail-m +base_model: {base_model} +prompt: "a photo of sks" +tags: +- stable-diffusion-inpainting +- stable-diffusion-inpainting-diffusers +- text-to-image +- diffusers +- realfill +inference: true +--- + """ + model_card = f""" +# RealFill - {repo_id} + +This is a realfill model derived from {base_model}. The weights were trained using [RealFill](https://realfill.github.io/). +You can find some example images in the following. \n +{img_str} +""" + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + +def log_validation( + text_encoder, + tokenizer, + unet, + args, + accelerator, + weight_dtype, + epoch, +): + logger.info(f"Running validation... \nGenerating {args.num_validation_images} images") + + # create pipeline (note: unet and vae are loaded again in float32) + pipeline = StableDiffusionInpaintPipeline.from_pretrained( + args.pretrained_model_name_or_path, + tokenizer=tokenizer, + revision=args.revision, + torch_dtype=weight_dtype, + ) + + # set `keep_fp32_wrapper` to True because we do not want to remove + # mixed precision hooks while we are still training + pipeline.unet = accelerator.unwrap_model(unet, keep_fp32_wrapper=True) + pipeline.text_encoder = accelerator.unwrap_model(text_encoder, keep_fp32_wrapper=True) + pipeline.scheduler = DPMSolverMultistepScheduler.from_config(pipeline.scheduler.config) + + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + # run inference + generator = None if args.seed is None else torch.Generator(device=accelerator.device).manual_seed(args.seed) + + target_dir = Path(args.train_data_dir) / "target" + target_image, target_mask = target_dir / "target.png", target_dir / "mask.png" + image, mask_image = Image.open(target_image), Image.open(target_mask) + + if image.mode != "RGB": + image = image.convert("RGB") + + images = [] + for _ in range(args.num_validation_images): + image = pipeline( + prompt="a photo of sks", + image=image, + mask_image=mask_image, + num_inference_steps=25, + guidance_scale=5, + generator=generator, + ).images[0] + images.append(image) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log({"validation": [wandb.Image(image, caption=str(i)) for i, image in enumerate(images)]}) + + del pipeline + torch.cuda.empty_cache() + + return images + + +def parse_args(input_args=None): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--tokenizer_name", + type=str, + default=None, + help="Pretrained tokenizer name or path if not the same as model_name", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + required=True, + help="A folder containing the training data of images.", + ) + parser.add_argument( + "--num_validation_images", + type=int, + default=4, + help="Number of images that should be generated during validation with `validation_conditioning`.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=100, + help=( + "Run realfill validation every X steps. RealFill validation consists of running the conditioning" + " `args.validation_conditioning` multiple times: `args.num_validation_images`." + ), + ) + parser.add_argument( + "--output_dir", + type=str, + default="realfill-model", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--train_batch_size", type=int, default=4, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=1) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints can be used both as final" + " checkpoints in case they are better than the last checkpoint, and are also suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--unet_learning_rate", + type=float, + default=2e-4, + help="Learning rate to use for unet.", + ) + parser.add_argument( + "--text_encoder_learning_rate", + type=float, + default=4e-5, + help="Learning rate to use for text encoder.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--lr_num_cycles", + type=int, + default=1, + help="Number of hard resets of the lr in cosine_with_restarts scheduler.", + ) + parser.add_argument("--lr_power", type=float, default=1.0, help="Power factor of the polynomial scheduler.") + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument( + "--wandb_key", + type=str, + default=None, + help=("If report to option is set to wandb, api-key for wandb used for login to wandb "), + ) + parser.add_argument( + "--wandb_project_name", + type=str, + default=None, + help=("If report to option is set to wandb, project name in wandb for log tracking "), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." + ) + parser.add_argument( + "--set_grads_to_none", + action="store_true", + help=( + "Save more memory by using setting grads to None instead of zero. Be aware, that this changes certain" + " behaviors, so disable this argument if it causes any problems. More info:" + " https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html" + ), + ) + parser.add_argument( + "--lora_rank", + type=int, + default=16, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--lora_alpha", + type=int, + default=27, + help=("The alpha constant of the LoRA update matrices."), + ) + parser.add_argument( + "--lora_dropout", + type=float, + default=0.0, + help="The dropout rate of the LoRA update matrices.", + ) + parser.add_argument( + "--lora_bias", + type=str, + default="none", + help="The bias type of the Lora update matrices. Must be 'none', 'all' or 'lora_only'.", + ) + + if input_args is not None: + args = parser.parse_args(input_args) + else: + args = parser.parse_args() + + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + return args + + +class RealFillDataset(Dataset): + """ + A dataset to prepare the training and conditioning images and + the masks with the dummy prompt for fine-tuning the model. + It pre-processes the images, masks and tokenizes the prompts. + """ + + def __init__( + self, + train_data_root, + tokenizer, + size=512, + ): + self.size = size + self.tokenizer = tokenizer + + self.ref_data_root = Path(train_data_root) / "ref" + self.target_image = Path(train_data_root) / "target" / "target.png" + self.target_mask = Path(train_data_root) / "target" / "mask.png" + if not (self.ref_data_root.exists() and self.target_image.exists() and self.target_mask.exists()): + raise ValueError("Train images root doesn't exists.") + + self.train_images_path = list(self.ref_data_root.iterdir()) + [self.target_image] + self.num_train_images = len(self.train_images_path) + self.train_prompt = "a photo of sks" + + self.transform = transforms_v2.Compose( + [ + transforms_v2.RandomResize(size, int(1.125 * size)), + transforms_v2.RandomCrop(size), + transforms_v2.ToImageTensor(), + transforms_v2.ConvertImageDtype(), + transforms_v2.Normalize([0.5], [0.5]), + ] + ) + + def __len__(self): + return self.num_train_images + + def __getitem__(self, index): + example = {} + + image = Image.open(self.train_images_path[index]) + image = exif_transpose(image) + + if not image.mode == "RGB": + image = image.convert("RGB") + + if index < len(self) - 1: + weighting = Image.new("L", image.size) + else: + weighting = Image.open(self.target_mask) + weighting = exif_transpose(weighting) + + image, weighting = self.transform(image, weighting) + example["images"], example["weightings"] = image, weighting < 0 + + if random.random() < 0.1: + example["masks"] = torch.ones_like(example["images"][0:1, :, :]) + else: + example["masks"] = make_mask(example["images"], self.size) + + example["conditioning_images"] = example["images"] * (example["masks"] < 0.5) + + train_prompt = "" if random.random() < 0.1 else self.train_prompt + example["prompt_ids"] = self.tokenizer( + train_prompt, + truncation=True, + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ).input_ids + + return example + + +def collate_fn(examples): + input_ids = [example["prompt_ids"] for example in examples] + images = [example["images"] for example in examples] + + masks = [example["masks"] for example in examples] + weightings = [example["weightings"] for example in examples] + conditioning_images = [example["conditioning_images"] for example in examples] + + images = torch.stack(images) + images = images.to(memory_format=torch.contiguous_format).float() + + masks = torch.stack(masks) + masks = masks.to(memory_format=torch.contiguous_format).float() + + weightings = torch.stack(weightings) + weightings = weightings.to(memory_format=torch.contiguous_format).float() + + conditioning_images = torch.stack(conditioning_images) + conditioning_images = conditioning_images.to(memory_format=torch.contiguous_format).float() + + input_ids = torch.cat(input_ids, dim=0) + + batch = { + "input_ids": input_ids, + "images": images, + "masks": masks, + "weightings": weightings, + "conditioning_images": conditioning_images, + } + return batch + + +def main(args): + logging_dir = Path(args.output_dir, args.logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_dir=logging_dir, + ) + + if args.report_to == "wandb": + if not is_wandb_available(): + raise ImportError("Make sure to install wandb if you want to use it for logging during training.") + + wandb.login(key=args.wandb_key) + wandb.init(project=args.wandb_project_name) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Load the tokenizer + if args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, revision=args.revision, use_fast=False) + elif args.pretrained_model_name_or_path: + tokenizer = AutoTokenizer.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="tokenizer", + revision=args.revision, + use_fast=False, + ) + + # Load scheduler and models + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + text_encoder = CLIPTextModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision + ) + vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision) + unet = UNet2DConditionModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision + ) + + config = LoraConfig( + r=args.lora_rank, + lora_alpha=args.lora_alpha, + target_modules=["to_k", "to_q", "to_v", "key", "query", "value"], + lora_dropout=args.lora_dropout, + bias=args.lora_bias, + ) + unet = get_peft_model(unet, config) + + config = LoraConfig( + r=args.lora_rank, + lora_alpha=args.lora_alpha, + target_modules=["k_proj", "q_proj", "v_proj"], + lora_dropout=args.lora_dropout, + bias=args.lora_bias, + ) + text_encoder = get_peft_model(text_encoder, config) + + vae.requires_grad_(False) + + if args.enable_xformers_memory_efficient_attention: + if is_xformers_available(): + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse("0.0.16"): + logger.warn( + "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." + ) + unet.enable_xformers_memory_efficient_attention() + else: + raise ValueError("xformers is not available. Make sure it is installed correctly") + + if args.gradient_checkpointing: + unet.enable_gradient_checkpointing() + text_encoder.gradient_checkpointing_enable() + + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + for model in models: + sub_dir = ( + "unet" + if isinstance(model.base_model.model, type(accelerator.unwrap_model(unet.base_model.model))) + else "text_encoder" + ) + model.save_pretrained(os.path.join(output_dir, sub_dir)) + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + def load_model_hook(models, input_dir): + while len(models) > 0: + # pop models so that they are not loaded again + model = models.pop() + + sub_dir = ( + "unet" + if isinstance(model.base_model.model, type(accelerator.unwrap_model(unet.base_model.model))) + else "text_encoder" + ) + model_cls = ( + UNet2DConditionModel + if isinstance(model.base_model.model, type(accelerator.unwrap_model(unet.base_model.model))) + else CLIPTextModel + ) + + load_model = model_cls.from_pretrained(args.pretrained_model_name_or_path, subfolder=sub_dir) + load_model = PeftModel.from_pretrained(load_model, input_dir, subfolder=sub_dir) + + model.load_state_dict(load_model.state_dict()) + del load_model + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.unet_learning_rate = ( + args.unet_learning_rate + * args.gradient_accumulation_steps + * args.train_batch_size + * accelerator.num_processes + ) + + args.text_encoder_learning_rate = ( + args.text_encoder_learning_rate + * args.gradient_accumulation_steps + * args.train_batch_size + * accelerator.num_processes + ) + + # Use 8-bit Adam for lower memory usage or to fine-tune the model in 16GB GPUs + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "To use 8-bit Adam, please install the bitsandbytes library: `pip install bitsandbytes`." + ) + + optimizer_class = bnb.optim.AdamW8bit + else: + optimizer_class = torch.optim.AdamW + + # Optimizer creation + optimizer = optimizer_class( + [ + {"params": unet.parameters(), "lr": args.unet_learning_rate}, + {"params": text_encoder.parameters(), "lr": args.text_encoder_learning_rate}, + ], + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Dataset and DataLoaders creation: + train_dataset = RealFillDataset( + train_data_root=args.train_data_dir, + tokenizer=tokenizer, + size=args.resolution, + ) + + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_size=args.train_batch_size, + shuffle=True, + collate_fn=collate_fn, + num_workers=1, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + num_cycles=args.lr_num_cycles, + power=args.lr_power, + ) + + # Prepare everything with our `accelerator`. + unet, text_encoder, optimizer, train_dataloader = accelerator.prepare( + unet, text_encoder, optimizer, train_dataloader + ) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora unet) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + + # Move vae to device and cast to weight_dtype + vae.to(accelerator.device, dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = vars(copy.deepcopy(args)) + accelerator.init_trackers("realfill", config=tracker_config) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num batches each epoch = {len(train_dataloader)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the mos recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + first_epoch = global_step // num_update_steps_per_epoch + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + for epoch in range(first_epoch, args.num_train_epochs): + unet.train() + text_encoder.train() + + for step, batch in enumerate(train_dataloader): + with accelerator.accumulate(unet, text_encoder): + # Convert images to latent space + latents = vae.encode(batch["images"].to(dtype=weight_dtype)).latent_dist.sample() + latents = latents * 0.18215 + + # Convert masked images to latent space + conditionings = vae.encode(batch["conditioning_images"].to(dtype=weight_dtype)).latent_dist.sample() + conditionings = conditionings * 0.18215 + + # Downsample mask and weighting so that they match with the latents + masks, size = batch["masks"].to(dtype=weight_dtype), latents.shape[2:] + masks = F.interpolate(masks, size=size) + + weightings = batch["weightings"].to(dtype=weight_dtype) + weightings = F.interpolate(weightings, size=size) + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + # Concatenate noisy latents, masks and conditionings to get inputs to unet + inputs = torch.cat([noisy_latents, masks, conditionings], dim=1) + + # Get the text embedding for conditioning + encoder_hidden_states = text_encoder(batch["input_ids"])[0] + + # Predict the noise residual + model_pred = unet(inputs, timesteps, encoder_hidden_states).sample + + # Compute the diffusion loss + assert noise_scheduler.config.prediction_type == "epsilon" + loss = (weightings * F.mse_loss(model_pred.float(), noise.float(), reduction="none")).mean() + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + params_to_clip = itertools.chain(unet.parameters(), text_encoder.parameters()) + accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad(set_to_none=args.set_grads_to_none) + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + if args.report_to == "wandb": + accelerator.print(progress_bar) + global_step += 1 + + if accelerator.is_main_process: + if global_step % args.checkpointing_steps == 0: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if global_step % args.validation_steps == 0: + log_validation( + text_encoder, + tokenizer, + unet, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"loss": loss.detach().item()} + progress_bar.set_postfix(**logs) + accelerator.log(logs, step=global_step) + + if global_step >= args.max_train_steps: + break + + # Save the lora layers + accelerator.wait_for_everyone() + if accelerator.is_main_process: + pipeline = StableDiffusionInpaintPipeline.from_pretrained( + args.pretrained_model_name_or_path, + unet=accelerator.unwrap_model(unet.merge_and_unload(), keep_fp32_wrapper=True), + text_encoder=accelerator.unwrap_model(text_encoder.merge_and_unload(), keep_fp32_wrapper=True), + revision=args.revision, + ) + + pipeline.save_pretrained(args.output_dir) + + # Final inference + images = log_validation( + text_encoder, + tokenizer, + unet, + args, + accelerator, + weight_dtype, + global_step, + ) + + if args.push_to_hub: + save_model_card( + repo_id, + images=images, + base_model=args.pretrained_model_name_or_path, + repo_folder=args.output_dir, + ) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + accelerator.end_training() + + +if __name__ == "__main__": + args = parse_args() + main(args) From 3fc10ded000d673768bf03195b0600f69af96a50 Mon Sep 17 00:00:00 2001 From: "Peter @sHTiF Stefcek" Date: Mon, 30 Oct 2023 16:46:44 +0100 Subject: [PATCH 09/28] add fix to be able use StableDiffusionXLAdapterPipeline.from_single_file (#5547) --- src/diffusers/loaders.py | 2 ++ .../pipelines/stable_diffusion/convert_from_ckpt.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/diffusers/loaders.py b/src/diffusers/loaders.py index 67043866be6e..87e0e164026f 100644 --- a/src/diffusers/loaders.py +++ b/src/diffusers/loaders.py @@ -2727,6 +2727,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): text_encoder = kwargs.pop("text_encoder", None) vae = kwargs.pop("vae", None) controlnet = kwargs.pop("controlnet", None) + adapter = kwargs.pop("adapter", None) tokenizer = kwargs.pop("tokenizer", None) torch_dtype = kwargs.pop("torch_dtype", None) @@ -2819,6 +2820,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): model_type=model_type, stable_unclip=stable_unclip, controlnet=controlnet, + adapter=adapter, from_safetensors=from_safetensors, extract_ema=extract_ema, image_size=image_size, diff --git a/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py b/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py index ffe81ea44a27..8c1d52ca83d8 100644 --- a/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +++ b/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py @@ -1145,6 +1145,7 @@ def download_from_original_stable_diffusion_ckpt( stable_unclip_prior: Optional[str] = None, clip_stats_path: Optional[str] = None, controlnet: Optional[bool] = None, + adapter: Optional[bool] = None, load_safety_checker: bool = True, pipeline_class: DiffusionPipeline = None, local_files_only=False, @@ -1723,6 +1724,18 @@ def download_from_original_stable_diffusion_ckpt( scheduler=scheduler, force_zeros_for_empty_prompt=True, ) + elif adapter: + pipe = pipeline_class( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + unet=unet, + adapter=adapter, + scheduler=scheduler, + force_zeros_for_empty_prompt=True, + ) else: pipe = pipeline_class( vae=vae, From ac7b1716b7941e67311b0a5bc56979317cc70f93 Mon Sep 17 00:00:00 2001 From: Cheng Lu Date: Tue, 31 Oct 2023 00:36:53 +0800 Subject: [PATCH 10/28] Stabilize DPM++, especially for SDXL and SDE-DPM++ (#5541) * stabilize dpmpp for sdxl by using euler at the final step * add lu's uniform logsnr time steps * add test * fix check_copies * fix tests --------- Co-authored-by: Patrick von Platen --- .../scheduling_dpmsolver_multistep.py | 34 +++++++++++++++++-- .../scheduling_dpmsolver_multistep_inverse.py | 10 ++++-- tests/schedulers/test_scheduler_dpm_multi.py | 11 ++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py index 6b1a43630fa6..b9183e6d80c8 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py @@ -117,9 +117,17 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): lower_order_final (`bool`, defaults to `True`): Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. use_karras_sigmas (`bool`, *optional*, defaults to `False`): Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, the sigmas are determined according to a sequence of noise levels {σi}. + use_lu_lambdas (`bool`, *optional*, defaults to `False`): + Whether to use the uniform-logSNR for step sizes proposed by Lu's DPM-Solver in the noise schedule during + the sampling process. If `True`, the sigmas and time steps are determined according to a sequence of + `lambda(t)`. lambda_min_clipped (`float`, defaults to `-inf`): Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the cosine (`squaredcos_cap_v2`) noise schedule. @@ -154,7 +162,9 @@ def __init__( algorithm_type: str = "dpmsolver++", solver_type: str = "midpoint", lower_order_final: bool = True, + euler_at_final: bool = False, use_karras_sigmas: Optional[bool] = False, + use_lu_lambdas: Optional[bool] = False, lambda_min_clipped: float = -float("inf"), variance_type: Optional[str] = None, timestep_spacing: str = "linspace", @@ -258,6 +268,12 @@ def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torc sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32) + elif self.config.use_lu_lambdas: + lambdas = np.flip(log_sigmas.copy()) + lambdas = self._convert_to_lu(in_lambdas=lambdas, num_inference_steps=num_inference_steps) + sigmas = np.exp(lambdas) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32) else: sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 @@ -354,6 +370,19 @@ def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho return sigmas + def _convert_to_lu(self, in_lambdas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Lu et al. (2022).""" + + lambda_min: float = in_lambdas[-1].item() + lambda_max: float = in_lambdas[0].item() + + rho = 1.0 # 1.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = lambda_min ** (1 / rho) + max_inv_rho = lambda_max ** (1 / rho) + lambdas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return lambdas + def convert_model_output( self, model_output: torch.FloatTensor, @@ -787,8 +816,9 @@ def step( if self.step_index is None: self._init_step_index(timestep) - lower_order_final = ( - (self.step_index == len(self.timesteps) - 1) and self.config.lower_order_final and len(self.timesteps) < 15 + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final or (self.config.lower_order_final and len(self.timesteps) < 15) ) lower_order_second = ( (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py index fa8f362bd3b5..0168b8405391 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py @@ -117,6 +117,10 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin): lower_order_final (`bool`, defaults to `True`): Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. use_karras_sigmas (`bool`, *optional*, defaults to `False`): Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, the sigmas are determined according to a sequence of noise levels {σi}. @@ -154,6 +158,7 @@ def __init__( algorithm_type: str = "dpmsolver++", solver_type: str = "midpoint", lower_order_final: bool = True, + euler_at_final: bool = False, use_karras_sigmas: Optional[bool] = False, lambda_min_clipped: float = -float("inf"), variance_type: Optional[str] = None, @@ -804,8 +809,9 @@ def step( if self.step_index is None: self._init_step_index(timestep) - lower_order_final = ( - (self.step_index == len(self.timesteps) - 1) and self.config.lower_order_final and len(self.timesteps) < 15 + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final or (self.config.lower_order_final and len(self.timesteps) < 15) ) lower_order_second = ( (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 diff --git a/tests/schedulers/test_scheduler_dpm_multi.py b/tests/schedulers/test_scheduler_dpm_multi.py index 6e6442e0daf6..7fe71941b4e7 100644 --- a/tests/schedulers/test_scheduler_dpm_multi.py +++ b/tests/schedulers/test_scheduler_dpm_multi.py @@ -29,6 +29,7 @@ def get_scheduler_config(self, **kwargs): "algorithm_type": "dpmsolver++", "solver_type": "midpoint", "lower_order_final": False, + "euler_at_final": False, "lambda_min_clipped": -float("inf"), "variance_type": None, } @@ -195,6 +196,10 @@ def test_lower_order_final(self): self.check_over_configs(lower_order_final=True) self.check_over_configs(lower_order_final=False) + def test_euler_at_final(self): + self.check_over_configs(euler_at_final=True) + self.check_over_configs(euler_at_final=False) + def test_lambda_min_clipped(self): self.check_over_configs(lambda_min_clipped=-float("inf")) self.check_over_configs(lambda_min_clipped=-5.1) @@ -258,6 +263,12 @@ def test_full_loop_with_karras_and_v_prediction(self): assert abs(result_mean.item() - 0.2096) < 1e-3 + def test_full_loop_with_lu_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_lu_lambdas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1554) < 1e-3 + def test_switch(self): # make sure that iterating over schedulers with same config names gives same results # for defaults From bb46be2f18c09aaefefd10f53c804c9ea604786f Mon Sep 17 00:00:00 2001 From: Aryan V S Date: Tue, 31 Oct 2023 00:02:11 +0530 Subject: [PATCH 11/28] Fix incorrect loading of custom pipeline (#5568) * update * update * update * update --- src/diffusers/pipelines/pipeline_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py index 512cf8d56718..8baafbaef115 100644 --- a/src/diffusers/pipelines/pipeline_utils.py +++ b/src/diffusers/pipelines/pipeline_utils.py @@ -1669,7 +1669,7 @@ def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: for component in folder_names: module_candidate = config_dict[component][0] - if module_candidate is None: + if module_candidate is None or not isinstance(module_candidate, str): continue candidate_file = os.path.join(component, module_candidate + ".py") From 32fea1cc9bd36e91e42558f978ea224f971f0b51 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 30 Oct 2023 19:35:46 +0100 Subject: [PATCH 12/28] [`core` / `PEFT` ]Bump transformers min version for PEFT integration (#5579) Update constants.py --- src/diffusers/utils/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/diffusers/utils/constants.py b/src/diffusers/utils/constants.py index 3023cb476fe0..a485498eb725 100644 --- a/src/diffusers/utils/constants.py +++ b/src/diffusers/utils/constants.py @@ -23,6 +23,7 @@ default_cache_path = HUGGINGFACE_HUB_CACHE MIN_PEFT_VERSION = "0.5.0" +MIN_TRANSFORMERS_VERSION = "4.33.3" CONFIG_NAME = "config.json" @@ -46,6 +47,6 @@ ) > version.parse(MIN_PEFT_VERSION) _required_transformers_version = is_transformers_available() and version.parse( version.parse(importlib.metadata.version("transformers")).base_version -) > version.parse("4.33") +) > version.parse(MIN_TRANSFORMERS_VERSION) USE_PEFT_BACKEND = _required_peft_version and _required_transformers_version From f0b2f6ce054c9f42e5515699da299f8ff69ed77f Mon Sep 17 00:00:00 2001 From: TimothyAlexisVass <55708319+TimothyAlexisVass@users.noreply.github.com> Date: Tue, 31 Oct 2023 07:09:08 +0100 Subject: [PATCH 13/28] Fix divide by zero RuntimeWarning (#5543) --- src/diffusers/schedulers/scheduling_deis_multistep.py | 2 +- src/diffusers/schedulers/scheduling_dpmsolver_multistep.py | 2 +- .../schedulers/scheduling_dpmsolver_multistep_inverse.py | 2 +- src/diffusers/schedulers/scheduling_dpmsolver_sde.py | 2 +- src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py | 2 +- src/diffusers/schedulers/scheduling_euler_discrete.py | 2 +- src/diffusers/schedulers/scheduling_heun_discrete.py | 2 +- .../schedulers/scheduling_k_dpm_2_ancestral_discrete.py | 2 +- src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py | 2 +- src/diffusers/schedulers/scheduling_lms_discrete.py | 2 +- src/diffusers/schedulers/scheduling_unipc_multistep.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/diffusers/schedulers/scheduling_deis_multistep.py b/src/diffusers/schedulers/scheduling_deis_multistep.py index a6afe744bd88..39763191bce1 100644 --- a/src/diffusers/schedulers/scheduling_deis_multistep.py +++ b/src/diffusers/schedulers/scheduling_deis_multistep.py @@ -293,7 +293,7 @@ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py index b9183e6d80c8..479a27de41ea 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py @@ -329,7 +329,7 @@ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py index 0168b8405391..1c0ea675bc18 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py @@ -328,7 +328,7 @@ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_sde.py b/src/diffusers/schedulers/scheduling_dpmsolver_sde.py index d39efbe724fb..60c6341a945b 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_sde.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_sde.py @@ -373,7 +373,7 @@ def t_fn(_sigma): # copied from diffusers.schedulers.scheduling_euler_discrete._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py b/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py index bb7dc21e6fdb..befc79c2f21c 100644 --- a/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py +++ b/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py @@ -327,7 +327,7 @@ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_euler_discrete.py b/src/diffusers/schedulers/scheduling_euler_discrete.py index 0875e1af3325..bc703a8f072c 100644 --- a/src/diffusers/schedulers/scheduling_euler_discrete.py +++ b/src/diffusers/schedulers/scheduling_euler_discrete.py @@ -278,7 +278,7 @@ def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.devic def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_heun_discrete.py b/src/diffusers/schedulers/scheduling_heun_discrete.py index a5827bbc8610..db5797f7d238 100644 --- a/src/diffusers/schedulers/scheduling_heun_discrete.py +++ b/src/diffusers/schedulers/scheduling_heun_discrete.py @@ -280,7 +280,7 @@ def set_timesteps( # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py b/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py index a0137b83fda1..115436c8e360 100644 --- a/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +++ b/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py @@ -301,7 +301,7 @@ def set_timesteps( # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py b/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py index ddea57e8c167..1c25738af274 100644 --- a/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py +++ b/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py @@ -312,7 +312,7 @@ def _init_step_index(self, timestep): # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_lms_discrete.py b/src/diffusers/schedulers/scheduling_lms_discrete.py index 9bee37d59ee1..05126377763e 100644 --- a/src/diffusers/schedulers/scheduling_lms_discrete.py +++ b/src/diffusers/schedulers/scheduling_lms_discrete.py @@ -305,7 +305,7 @@ def _init_step_index(self, timestep): # copied from diffusers.schedulers.scheduling_euler_discrete._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] diff --git a/src/diffusers/schedulers/scheduling_unipc_multistep.py b/src/diffusers/schedulers/scheduling_unipc_multistep.py index 741b03b6d3a2..3bd7d2931764 100644 --- a/src/diffusers/schedulers/scheduling_unipc_multistep.py +++ b/src/diffusers/schedulers/scheduling_unipc_multistep.py @@ -307,7 +307,7 @@ def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t def _sigma_to_t(self, sigma, log_sigmas): # get log sigma - log_sigma = np.log(sigma) + log_sigma = np.log(np.maximum(sigma, 1e-10)) # get distribution dists = log_sigma - log_sigmas[:, np.newaxis] From ed00ead3451c06c26b776bf373c383e572ba6cc4 Mon Sep 17 00:00:00 2001 From: Jincheng Miao Date: Tue, 31 Oct 2023 14:24:16 +0800 Subject: [PATCH 14/28] [Community Pipelines] add textual inversion support for stable_diffusion_ipex (#5571) --- examples/community/stable_diffusion_ipex.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/community/stable_diffusion_ipex.py b/examples/community/stable_diffusion_ipex.py index 2f8131d6cbc0..58fe362f4a2f 100644 --- a/examples/community/stable_diffusion_ipex.py +++ b/examples/community/stable_diffusion_ipex.py @@ -21,6 +21,7 @@ from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer from diffusers.configuration_utils import FrozenDict +from diffusers.loaders import TextualInversionLoaderMixin from diffusers.models import AutoencoderKL, UNet2DConditionModel from diffusers.pipeline_utils import DiffusionPipeline from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput @@ -61,7 +62,7 @@ """ -class StableDiffusionIPEXPipeline(DiffusionPipeline): +class StableDiffusionIPEXPipeline(DiffusionPipeline, TextualInversionLoaderMixin): r""" Pipeline for text-to-image generation using Stable Diffusion on IPEX. @@ -454,6 +455,10 @@ def _encode_prompt( batch_size = prompt_embeds.shape[0] if prompt_embeds is None: + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + text_inputs = self.tokenizer( prompt, padding="max_length", @@ -514,6 +519,10 @@ def _encode_prompt( else: uncond_tokens = negative_prompt + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + max_length = prompt_embeds.shape[1] uncond_input = self.tokenizer( uncond_tokens, From ce9484b139014654164c124defd1e96a4767757b Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Mon, 30 Oct 2023 23:06:16 -1000 Subject: [PATCH 15/28] fix a mistake in text2image training script for kandinsky2.2 (#5244) fix Co-authored-by: yiyixuxu --- .../text_to_image/train_text_to_image_lora_prior.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py b/examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py index 7305137218ef..3656b480e9bb 100644 --- a/examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py +++ b/examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py @@ -682,7 +682,7 @@ def collate_fn(examples): # Backpropagate accelerator.backward(loss) if accelerator.sync_gradients: - accelerator.clip_grad_norm_(prior.parameters(), args.max_grad_norm) + accelerator.clip_grad_norm_(lora_layers.parameters(), args.max_grad_norm) optimizer.step() lr_scheduler.step() optimizer.zero_grad() From f1d052c5b8a4401e0e60352ddfeaafbb203e5bbf Mon Sep 17 00:00:00 2001 From: Dhruv Nair Date: Tue, 31 Oct 2023 15:02:10 +0530 Subject: [PATCH 16/28] Update docker image for xformers (#5597) update docker image for xformers --- docker/diffusers-pytorch-xformers-cuda/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/diffusers-pytorch-xformers-cuda/Dockerfile b/docker/diffusers-pytorch-xformers-cuda/Dockerfile index 95fe933798bc..003f8e1165a1 100644 --- a/docker/diffusers-pytorch-xformers-cuda/Dockerfile +++ b/docker/diffusers-pytorch-xformers-cuda/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu20.04 +FROM nvidia/cuda:12.1.0-runtime-ubuntu20.04 LABEL maintainer="Hugging Face" LABEL repository="diffusers" @@ -25,8 +25,8 @@ ENV PATH="/opt/venv/bin:$PATH" # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) RUN python3 -m pip install --no-cache-dir --upgrade pip && \ python3 -m pip install --no-cache-dir \ - torch==2.0.1 \ - torchvision==0.15.2 \ + torch \ + torchvision \ torchaudio \ invisible_watermark && \ python3 -m pip install --no-cache-dir \ From 442017ccc877279bcf24fbe92f92d3d0def191b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Tolga=20Cang=C3=B6z?= <46008593+standardAI@users.noreply.github.com> Date: Tue, 31 Oct 2023 20:04:08 +0300 Subject: [PATCH 17/28] [Docs] Fix typos (#5583) * Add Copyright info * Fix typos, improve, update * Update deepfloyd_if.md * Update ldm3d_diffusion.md * Update opt_overview.md --- .../en/api/pipelines/paint_by_example.md | 2 +- .../stable_diffusion/ldm3d_diffusion.md | 2 +- docs/source/en/api/pipelines/unclip.md | 6 ++--- docs/source/en/optimization/opt_overview.md | 4 ++-- .../en/using-diffusers/control_brightness.md | 12 ++++++++++ docs/source/en/using-diffusers/controlnet.md | 12 ++++++++++ docs/source/en/using-diffusers/diffedit.md | 12 ++++++++++ .../source/en/using-diffusers/distilled_sd.md | 12 ++++++++++ docs/source/en/using-diffusers/freeu.md | 12 ++++++++++ .../en/using-diffusers/pipeline_overview.md | 2 +- docs/source/en/using-diffusers/sdxl.md | 12 ++++++++++ docs/source/en/using-diffusers/shap-e.md | 12 ++++++++++ .../stable_diffusion_jax_how_to.md | 12 ++++++++++ .../textual_inversion_inference.md | 12 ++++++++++ examples/README.md | 2 +- examples/textual_inversion/README.md | 24 +++++++++++-------- 16 files changed, 131 insertions(+), 19 deletions(-) diff --git a/docs/source/en/api/pipelines/paint_by_example.md b/docs/source/en/api/pipelines/paint_by_example.md index a6d3c255e3dd..d04a378a09d3 100644 --- a/docs/source/en/api/pipelines/paint_by_example.md +++ b/docs/source/en/api/pipelines/paint_by_example.md @@ -10,7 +10,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# PaintByExample +# Paint By Example [Paint by Example: Exemplar-based Image Editing with Diffusion Models](https://huggingface.co/papers/2211.13227) is by Binxin Yang, Shuyang Gu, Bo Zhang, Ting Zhang, Xuejin Chen, Xiaoyan Sun, Dong Chen, Fang Wen. diff --git a/docs/source/en/api/pipelines/stable_diffusion/ldm3d_diffusion.md b/docs/source/en/api/pipelines/stable_diffusion/ldm3d_diffusion.md index 9d70ab4f88e6..2e489c0eeb7c 100644 --- a/docs/source/en/api/pipelines/stable_diffusion/ldm3d_diffusion.md +++ b/docs/source/en/api/pipelines/stable_diffusion/ldm3d_diffusion.md @@ -12,7 +12,7 @@ specific language governing permissions and limitations under the License. # Text-to-(RGB, depth) -LDM3D was proposed in [LDM3D: Latent Diffusion Model for 3D](https://huggingface.co/papers/2305.10853) by Gabriela Ben Melech Stan, Diana Wofk, Scottie Fox, Alex Redden, Will Saxton, Jean Yu, Estelle Aflalo, Shao-Yen Tseng, Fabio Nonato, Matthias Muller, and Vasudev Lal. LDM3D generates an image and a depth map from a given text prompt unlike the existing text-to-image diffusion models such as [Stable Diffusion](./stable_diffusion/overview) which only generates an image. With almost the same number of parameters, LDM3D achieves to create a latent space that can compress both the RGB images and the depth maps. +LDM3D was proposed in [LDM3D: Latent Diffusion Model for 3D](https://huggingface.co/papers/2305.10853) by Gabriela Ben Melech Stan, Diana Wofk, Scottie Fox, Alex Redden, Will Saxton, Jean Yu, Estelle Aflalo, Shao-Yen Tseng, Fabio Nonato, Matthias Muller, and Vasudev Lal. LDM3D generates an image and a depth map from a given text prompt unlike the existing text-to-image diffusion models such as [Stable Diffusion](./overview) which only generates an image. With almost the same number of parameters, LDM3D achieves to create a latent space that can compress both the RGB images and the depth maps. The abstract from the paper is: diff --git a/docs/source/en/api/pipelines/unclip.md b/docs/source/en/api/pipelines/unclip.md index 74258b7f7026..0cb5dc54dc29 100644 --- a/docs/source/en/api/pipelines/unclip.md +++ b/docs/source/en/api/pipelines/unclip.md @@ -7,9 +7,9 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# UnCLIP +# unCLIP -[Hierarchical Text-Conditional Image Generation with CLIP Latents](https://huggingface.co/papers/2204.06125) is by Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen. The UnCLIP model in 🤗 Diffusers comes from kakaobrain's [karlo]((https://github.com/kakaobrain/karlo)). +[Hierarchical Text-Conditional Image Generation with CLIP Latents](https://huggingface.co/papers/2204.06125) is by Aditya Ramesh, Prafulla Dhariwal, Alex Nichol, Casey Chu, Mark Chen. The unCLIP model in 🤗 Diffusers comes from kakaobrain's [karlo]((https://github.com/kakaobrain/karlo)). The abstract from the paper is following: @@ -34,4 +34,4 @@ Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers) - __call__ ## ImagePipelineOutput -[[autodoc]] pipelines.ImagePipelineOutput \ No newline at end of file +[[autodoc]] pipelines.ImagePipelineOutput diff --git a/docs/source/en/optimization/opt_overview.md b/docs/source/en/optimization/opt_overview.md index 1f809bb011ce..3a458291ce5b 100644 --- a/docs/source/en/optimization/opt_overview.md +++ b/docs/source/en/optimization/opt_overview.md @@ -12,6 +12,6 @@ specific language governing permissions and limitations under the License. # Overview -Generating high-quality outputs is computationally intensive, especially during each iterative step where you go from a noisy output to a less noisy output. One of 🤗 Diffuser's goal is to make this technology widely accessible to everyone, which includes enabling fast inference on consumer and specialized hardware. +Generating high-quality outputs is computationally intensive, especially during each iterative step where you go from a noisy output to a less noisy output. One of 🤗 Diffuser's goals is to make this technology widely accessible to everyone, which includes enabling fast inference on consumer and specialized hardware. -This section will cover tips and tricks - like half-precision weights and sliced attention - for optimizing inference speed and reducing memory-consumption. You'll also learn how to speed up your PyTorch code with [`torch.compile`](https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) or [ONNX Runtime](https://onnxruntime.ai/docs/), and enable memory-efficient attention with [xFormers](https://facebookresearch.github.io/xformers/). There are also guides for running inference on specific hardware like Apple Silicon, and Intel or Habana processors. \ No newline at end of file +This section will cover tips and tricks - like half-precision weights and sliced attention - for optimizing inference speed and reducing memory-consumption. You'll also learn how to speed up your PyTorch code with [`torch.compile`](https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) or [ONNX Runtime](https://onnxruntime.ai/docs/), and enable memory-efficient attention with [xFormers](https://facebookresearch.github.io/xformers/). There are also guides for running inference on specific hardware like Apple Silicon, and Intel or Habana processors. diff --git a/docs/source/en/using-diffusers/control_brightness.md b/docs/source/en/using-diffusers/control_brightness.md index c56c757bb1bc..17c107ba57b8 100644 --- a/docs/source/en/using-diffusers/control_brightness.md +++ b/docs/source/en/using-diffusers/control_brightness.md @@ -1,3 +1,15 @@ + + # Control image brightness The Stable Diffusion pipeline is mediocre at generating images that are either very bright or dark as explained in the [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) paper. The solutions proposed in the paper are currently implemented in the [`DDIMScheduler`] which you can use to improve the lighting in your images. diff --git a/docs/source/en/using-diffusers/controlnet.md b/docs/source/en/using-diffusers/controlnet.md index 9af2806672be..71fd3c7a307e 100644 --- a/docs/source/en/using-diffusers/controlnet.md +++ b/docs/source/en/using-diffusers/controlnet.md @@ -1,3 +1,15 @@ + + # ControlNet ControlNet is a type of model for controlling image diffusion models by conditioning the model with an additional input image. There are many types of conditioning inputs (canny edge, user sketching, human pose, depth, and more) you can use to control a diffusion model. This is hugely useful because it affords you greater control over image generation, making it easier to generate specific images without experimenting with different text prompts or denoising values as much. diff --git a/docs/source/en/using-diffusers/diffedit.md b/docs/source/en/using-diffusers/diffedit.md index 4c32eb4c482b..1c4a347e7396 100644 --- a/docs/source/en/using-diffusers/diffedit.md +++ b/docs/source/en/using-diffusers/diffedit.md @@ -1,3 +1,15 @@ + + # DiffEdit [[open-in-colab]] diff --git a/docs/source/en/using-diffusers/distilled_sd.md b/docs/source/en/using-diffusers/distilled_sd.md index 7653300b92ab..2dd96d98861d 100644 --- a/docs/source/en/using-diffusers/distilled_sd.md +++ b/docs/source/en/using-diffusers/distilled_sd.md @@ -1,3 +1,15 @@ + + # Distilled Stable Diffusion inference [[open-in-colab]] diff --git a/docs/source/en/using-diffusers/freeu.md b/docs/source/en/using-diffusers/freeu.md index 6c23ec754382..4f3c64096705 100644 --- a/docs/source/en/using-diffusers/freeu.md +++ b/docs/source/en/using-diffusers/freeu.md @@ -1,3 +1,15 @@ + + # Improve generation quality with FreeU [[open-in-colab]] diff --git a/docs/source/en/using-diffusers/pipeline_overview.md b/docs/source/en/using-diffusers/pipeline_overview.md index 6d3ee7cc61ce..292ce51d322a 100644 --- a/docs/source/en/using-diffusers/pipeline_overview.md +++ b/docs/source/en/using-diffusers/pipeline_overview.md @@ -14,4 +14,4 @@ specific language governing permissions and limitations under the License. A pipeline is an end-to-end class that provides a quick and easy way to use a diffusion system for inference by bundling independently trained models and schedulers together. Certain combinations of models and schedulers define specific pipeline types, like [`StableDiffusionXLPipeline`] or [`StableDiffusionControlNetPipeline`], with specific capabilities. All pipeline types inherit from the base [`DiffusionPipeline`] class; pass it any checkpoint, and it'll automatically detect the pipeline type and load the necessary components. -This section demonstrates how to use specific pipelines such as Stable Diffusion XL, ControlNet, and DiffEdit. You'll also learn how to use a distilled version of the Stable Diffusion model to speed up inference, how to create reproducible pipelines, and how to use and contribute community pipelines. \ No newline at end of file +This section demonstrates how to use specific pipelines such as Stable Diffusion XL, ControlNet, and DiffEdit. You'll also learn how to use a distilled version of the Stable Diffusion model to speed up inference, how to create reproducible pipelines, and how to use and contribute community pipelines. diff --git a/docs/source/en/using-diffusers/sdxl.md b/docs/source/en/using-diffusers/sdxl.md index 36286ecad863..1016c57ca0ec 100644 --- a/docs/source/en/using-diffusers/sdxl.md +++ b/docs/source/en/using-diffusers/sdxl.md @@ -1,3 +1,15 @@ + + # Stable Diffusion XL [[open-in-colab]] diff --git a/docs/source/en/using-diffusers/shap-e.md b/docs/source/en/using-diffusers/shap-e.md index 68542bf56773..b5ba7923049d 100644 --- a/docs/source/en/using-diffusers/shap-e.md +++ b/docs/source/en/using-diffusers/shap-e.md @@ -1,3 +1,15 @@ + + # Shap-E [[open-in-colab]] diff --git a/docs/source/en/using-diffusers/stable_diffusion_jax_how_to.md b/docs/source/en/using-diffusers/stable_diffusion_jax_how_to.md index d62ce0bf91bf..9cf82907180c 100644 --- a/docs/source/en/using-diffusers/stable_diffusion_jax_how_to.md +++ b/docs/source/en/using-diffusers/stable_diffusion_jax_how_to.md @@ -1,3 +1,15 @@ + + # JAX/Flax [[open-in-colab]] diff --git a/docs/source/en/using-diffusers/textual_inversion_inference.md b/docs/source/en/using-diffusers/textual_inversion_inference.md index 821b8ec6745a..6e690c62f76a 100644 --- a/docs/source/en/using-diffusers/textual_inversion_inference.md +++ b/docs/source/en/using-diffusers/textual_inversion_inference.md @@ -1,3 +1,15 @@ + + # Textual inversion [[open-in-colab]] diff --git a/examples/README.md b/examples/README.md index 9566e68fc51d..f0d8a6bb57f0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,7 @@ Diffusers examples are a collection of scripts to demonstrate how to effectively for a variety of use cases involving training or fine-tuning. **Note**: If you are looking for **official** examples on how to use `diffusers` for inference, -please have a look at [src/diffusers/pipelines](https://github.com/huggingface/diffusers/tree/main/src/diffusers/pipelines) +please have a look at [src/diffusers/pipelines](https://github.com/huggingface/diffusers/tree/main/src/diffusers/pipelines). Our examples aspire to be **self-contained**, **easy-to-tweak**, **beginner-friendly** and for **one-purpose-only**. More specifically, this means: diff --git a/examples/textual_inversion/README.md b/examples/textual_inversion/README.md index 21bca526b5d2..0a1d8a459fc6 100644 --- a/examples/textual_inversion/README.md +++ b/examples/textual_inversion/README.md @@ -25,12 +25,12 @@ cd diffusers pip install . ``` -Then cd in the example folder and run +Then cd in the example folder and run: ```bash pip install -r requirements.txt ``` -And initialize an [🤗Accelerate](https://github.com/huggingface/accelerate/) environment with: +And initialize an [🤗 Accelerate](https://github.com/huggingface/accelerate/) environment with: ```bash accelerate config @@ -56,7 +56,7 @@ snapshot_download("diffusers/cat_toy_example", local_dir=local_dir, repo_type="d ``` This will be our training data. -Now we can launch the training using +Now we can launch the training using: **___Note: Change the `resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model.___** @@ -68,12 +68,14 @@ accelerate launch textual_inversion.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATA_DIR \ --learnable_property="object" \ - --placeholder_token="" --initializer_token="toy" \ + --placeholder_token="" \ + --initializer_token="toy" \ --resolution=512 \ --train_batch_size=1 \ --gradient_accumulation_steps=4 \ --max_train_steps=3000 \ - --learning_rate=5.0e-04 --scale_lr \ + --learning_rate=5.0e-04 \ + --scale_lr \ --lr_scheduler="constant" \ --lr_warmup_steps=0 \ --push_to_hub \ @@ -85,10 +87,10 @@ A full training run takes ~1 hour on one V100 GPU. **Note**: As described in [the official paper](https://arxiv.org/abs/2208.01618) only one embedding vector is used for the placeholder token, *e.g.* `""`. However, one can also add multiple embedding vectors for the placeholder token -to inclease the number of fine-tuneable parameters. This can help the model to learn -more complex details. To use multiple embedding vectors, you can should define `--num_vectors` +to increase the number of fine-tuneable parameters. This can help the model to learn +more complex details. To use multiple embedding vectors, you should define `--num_vectors` to a number larger than one, *e.g.*: -``` +```bash --num_vectors 5 ``` @@ -131,11 +133,13 @@ python textual_inversion_flax.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --train_data_dir=$DATA_DIR \ --learnable_property="object" \ - --placeholder_token="" --initializer_token="toy" \ + --placeholder_token="" \ + --initializer_token="toy" \ --resolution=512 \ --train_batch_size=1 \ --max_train_steps=3000 \ - --learning_rate=5.0e-04 --scale_lr \ + --learning_rate=5.0e-04 \ + --scale_lr \ --output_dir="textual_inversion_cat" ``` It should be at least 70% faster than the PyTorch script with the same configuration. From 5c75a5fbc421e3126c796822c274b496b6ea71ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Tolga=20Cang=C3=B6z?= <46008593+standardAI@users.noreply.github.com> Date: Wed, 1 Nov 2023 20:40:47 +0300 Subject: [PATCH 18/28] [Docs] Fix typos, improve, update at Tutorials page (#5586) * Fix typos, improve, update * Update autopipeline.md * Update docs/source/en/tutorials/using_peft_for_inference.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/tutorials/using_peft_for_inference.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/source/en/tutorials/using_peft_for_inference.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --------- Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/tutorials/autopipeline.md | 30 +++++++++++++++++-- docs/source/en/tutorials/basic_training.md | 6 ++-- docs/source/en/tutorials/tutorial_overview.md | 2 +- .../en/tutorials/using_peft_for_inference.md | 26 ++++++++-------- .../en/using-diffusers/write_own_pipeline.md | 2 +- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/docs/source/en/tutorials/autopipeline.md b/docs/source/en/tutorials/autopipeline.md index 973a83c73eb1..fcc6f5300eab 100644 --- a/docs/source/en/tutorials/autopipeline.md +++ b/docs/source/en/tutorials/autopipeline.md @@ -1,3 +1,15 @@ + + # AutoPipeline 🤗 Diffusers is able to complete many different tasks, and you can often reuse the same pretrained weights for multiple tasks such as text-to-image, image-to-image, and inpainting. If you're new to the library and diffusion models though, it may be difficult to know which pipeline to use for a task. For example, if you're using the [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) checkpoint for text-to-image, you might not know that you could also use it for image-to-image and inpainting by loading the checkpoint with the [`StableDiffusionImg2ImgPipeline`] and [`StableDiffusionInpaintPipeline`] classes respectively. @@ -6,7 +18,7 @@ The `AutoPipeline` class is designed to simplify the variety of pipelines in -Take a look at the [AutoPipeline](./pipelines/auto_pipeline) reference to see which tasks are supported. Currently, it supports text-to-image, image-to-image, and inpainting. +Take a look at the [AutoPipeline](../api/pipelines/auto_pipeline) reference to see which tasks are supported. Currently, it supports text-to-image, image-to-image, and inpainting. @@ -26,6 +38,7 @@ pipeline = AutoPipelineForText2Image.from_pretrained( prompt = "peasant and dragon combat, wood cutting style, viking era, bevel with rune" image = pipeline(prompt, num_inference_steps=25).images[0] +image ```
@@ -35,12 +48,16 @@ image = pipeline(prompt, num_inference_steps=25).images[0] Under the hood, [`AutoPipelineForText2Image`]: 1. automatically detects a `"stable-diffusion"` class from the [`model_index.json`](https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/model_index.json) file -2. loads the corresponding text-to-image [`StableDiffusionPipline`] based on the `"stable-diffusion"` class name +2. loads the corresponding text-to-image [`StableDiffusionPipeline`] based on the `"stable-diffusion"` class name Likewise, for image-to-image, [`AutoPipelineForImage2Image`] detects a `"stable-diffusion"` checkpoint from the `model_index.json` file and it'll load the corresponding [`StableDiffusionImg2ImgPipeline`] behind the scenes. You can also pass any additional arguments specific to the pipeline class such as `strength`, which determines the amount of noise or variation added to an input image: ```py from diffusers import AutoPipelineForImage2Image +import torch +import requests +from PIL import Image +from io import BytesIO pipeline = AutoPipelineForImage2Image.from_pretrained( "runwayml/stable-diffusion-v1-5", @@ -56,6 +73,7 @@ image = Image.open(BytesIO(response.content)).convert("RGB") image.thumbnail((768, 768)) image = pipeline(prompt, image, num_inference_steps=200, strength=0.75, guidance_scale=10.5).images[0] +image ```
@@ -67,6 +85,7 @@ And if you want to do inpainting, then [`AutoPipelineForInpainting`] loads the u ```py from diffusers import AutoPipelineForInpainting from diffusers.utils import load_image +import torch pipeline = AutoPipelineForInpainting.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, use_safetensors=True @@ -80,6 +99,7 @@ mask_image = load_image(mask_url).convert("RGB") prompt = "A majestic tiger sitting on a bench" image = pipeline(prompt, image=init_image, mask_image=mask_image, num_inference_steps=50, strength=0.80).images[0] +image ```
@@ -106,6 +126,7 @@ The [`~AutoPipelineForImage2Image.from_pipe`] method detects the original pipeli ```py from diffusers import AutoPipelineForText2Image, AutoPipelineForImage2Image +import torch pipeline_text2img = AutoPipelineForText2Image.from_pretrained( "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16, use_safetensors=True @@ -126,6 +147,7 @@ If you passed an optional argument - like disabling the safety checker - to the ```py from diffusers import AutoPipelineForText2Image, AutoPipelineForImage2Image +import torch pipeline_text2img = AutoPipelineForText2Image.from_pretrained( "runwayml/stable-diffusion-v1-5", @@ -135,7 +157,7 @@ pipeline_text2img = AutoPipelineForText2Image.from_pretrained( ).to("cuda") pipeline_img2img = AutoPipelineForImage2Image.from_pipe(pipeline_text2img) -print(pipe.config.requires_safety_checker) +print(pipeline_img2img.config.requires_safety_checker) "False" ``` @@ -143,4 +165,6 @@ You can overwrite any of the arguments and even configuration from the original ```py pipeline_img2img = AutoPipelineForImage2Image.from_pipe(pipeline_text2img, requires_safety_checker=True, strength=0.3) +print(pipeline_img2img.config.requires_safety_checker) +"True" ``` diff --git a/docs/source/en/tutorials/basic_training.md b/docs/source/en/tutorials/basic_training.md index 3a9366baf84a..3b545cdf572e 100644 --- a/docs/source/en/tutorials/basic_training.md +++ b/docs/source/en/tutorials/basic_training.md @@ -31,7 +31,7 @@ Before you begin, make sure you have 🤗 Datasets installed to load and preproc #!pip install diffusers[training] ``` -We encourage you to share your model with the community, and in order to do that, you'll need to login to your Hugging Face account (create one [here](https://hf.co/join) if you don't already have one!). You can login from a notebook and enter your token when prompted: +We encourage you to share your model with the community, and in order to do that, you'll need to login to your Hugging Face account (create one [here](https://hf.co/join) if you don't already have one!). You can login from a notebook and enter your token when prompted. Make sure your token has the write role. ```py >>> from huggingface_hub import notebook_login @@ -59,7 +59,6 @@ For convenience, create a `TrainingConfig` class containing the training hyperpa ```py >>> from dataclasses import dataclass - >>> @dataclass ... class TrainingConfig: ... image_size = 128 # the generated image resolution @@ -75,6 +74,7 @@ For convenience, create a `TrainingConfig` class containing the training hyperpa ... output_dir = "ddpm-butterflies-128" # the model name locally and on the HF Hub ... push_to_hub = True # whether to upload the saved model to the HF Hub +... hub_model_id = "/" # the name of the repository to create on the HF Hub ... hub_private_repo = False ... overwrite_output_dir = True # overwrite the old model when re-running the notebook ... seed = 0 @@ -253,10 +253,8 @@ Then, you'll need a way to evaluate the model. For evaluation, you can use the [ ```py >>> from diffusers import DDPMPipeline >>> from diffusers.utils import make_image_grid ->>> import math >>> import os - >>> def evaluate(config, epoch, pipeline): ... # Sample some images from random noise (this is the backward diffusion process). ... # The default pipeline output type is `List[PIL.Image]` diff --git a/docs/source/en/tutorials/tutorial_overview.md b/docs/source/en/tutorials/tutorial_overview.md index 0cec9a317ddb..85c30256ec89 100644 --- a/docs/source/en/tutorials/tutorial_overview.md +++ b/docs/source/en/tutorials/tutorial_overview.md @@ -20,4 +20,4 @@ After completing the tutorials, you'll have gained the necessary skills to start Feel free to join our community on [Discord](https://discord.com/invite/JfAtkvEtRb) or the [forums](https://discuss.huggingface.co/c/discussion-related-to-httpsgithubcomhuggingfacediffusers/63) to connect and collaborate with other users and developers! -Let's start diffusing! 🧨 \ No newline at end of file +Let's start diffusing! 🧨 diff --git a/docs/source/en/tutorials/using_peft_for_inference.md b/docs/source/en/tutorials/using_peft_for_inference.md index 4629cf8ba43c..2e3337519caa 100644 --- a/docs/source/en/tutorials/using_peft_for_inference.md +++ b/docs/source/en/tutorials/using_peft_for_inference.md @@ -10,11 +10,11 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -[[open-in-colab]] +[[open-in-colab]] # Inference with PEFT -There are many adapters trained in different styles to achieve different effects. You can even combine multiple adapters to create new and unique images. With the 🤗 [PEFT](https://huggingface.co/docs/peft/index) integration in 🤗 Diffusers, it is really easy to load and manage adapters for inference. In this guide, you'll learn how to use different adapters with [Stable Diffusion XL (SDXL)](./pipelines/stable_diffusion/stable_diffusion_xl) for inference. +There are many adapters trained in different styles to achieve different effects. You can even combine multiple adapters to create new and unique images. With the 🤗 [PEFT](https://huggingface.co/docs/peft/index) integration in 🤗 Diffusers, it is really easy to load and manage adapters for inference. In this guide, you'll learn how to use different adapters with [Stable Diffusion XL (SDXL)](../api/pipelines/stable_diffusion/stable_diffusion_xl) for inference. Throughout this guide, you'll use LoRA as the main adapter technique, so we'll use the terms LoRA and adapter interchangeably. You should have some familiarity with LoRA, and if you don't, we welcome you to check out the [LoRA guide](https://huggingface.co/docs/peft/conceptual_guides/lora). @@ -63,7 +63,7 @@ image With the `adapter_name` parameter, it is really easy to use another adapter for inference! Load the [nerijs/pixel-art-xl](https://huggingface.co/nerijs/pixel-art-xl) adapter that has been fine-tuned to generate pixel art images, and let's call it `"pixel"`. -The pipeline automatically sets the first loaded adapter (`"toy"`) as the active adapter. But you can activate the `"pixel"` adapter with the [`~diffusers.loaders.set_adapters`] method as shown below: +The pipeline automatically sets the first loaded adapter (`"toy"`) as the active adapter. But you can activate the `"pixel"` adapter with the [`~diffusers.loaders.UNet2DConditionLoadersMixin.set_adapters`] method as shown below: ```python pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") @@ -86,7 +86,7 @@ image You can also perform multi-adapter inference where you combine different adapter checkpoints for inference. -Once again, use the [`~diffusers.loaders.set_adapters`] method to activate two LoRA checkpoints and specify the weight for how the checkpoints should be combined. +Once again, use the [`~diffusers.loaders.UNet2DConditionLoadersMixin.set_adapters`] method to activate two LoRA checkpoints and specify the weight for how the checkpoints should be combined. ```python pipe.set_adapters(["pixel", "toy"], adapter_weights=[0.5, 1.0]) @@ -116,7 +116,7 @@ image Impressive! As you can see, the model was able to generate an image that mixes the characteristics of both adapters. -If you want to go back to using only one adapter, use the [`~diffusers.loaders.set_adapters`] method to activate the `"toy"` adapter: +If you want to go back to using only one adapter, use the [`~diffusers.loaders.UNet2DConditionLoadersMixin.set_adapters`] method to activate the `"toy"` adapter: ```python # First, set the adapter. @@ -134,7 +134,7 @@ image ![toy-face-again](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/peft_integration/diffusers_peft_lora_inference_18_1.png) -If you want to switch to only the base model, disable all LoRAs with the [`~diffusers.loaders.disable_lora`] method. +If you want to switch to only the base model, disable all LoRAs with the [`~diffusers.loaders.UNet2DConditionLoadersMixin.disable_lora`] method. ```python @@ -150,16 +150,18 @@ image ## Monitoring active adapters -You have attached multiple adapters in this tutorial, and if you're feeling a bit lost on what adapters have been attached to the pipeline's components, you can easily check the list of active adapters using the [`~diffusers.loaders.get_active_adapters`] method: +You have attached multiple adapters in this tutorial, and if you're feeling a bit lost on what adapters have been attached to the pipeline's components, you can easily check the list of active adapters using the [`~diffusers.loaders.LoraLoaderMixin.get_active_adapters`] method: -```python +```py active_adapters = pipe.get_active_adapters() ->>> ["toy", "pixel"] +active_adapters +["toy", "pixel"] ``` -You can also get the active adapters of each pipeline component with [`~diffusers.loaders.get_list_adapters`]: +You can also get the active adapters of each pipeline component with [`~diffusers.loaders.LoraLoaderMixin.get_list_adapters`]: -```python +```py list_adapters_component_wise = pipe.get_list_adapters() ->>> {"text_encoder": ["toy", "pixel"], "unet": ["toy", "pixel"], "text_encoder_2": ["toy", "pixel"]} +list_adapters_component_wise +{"text_encoder": ["toy", "pixel"], "unet": ["toy", "pixel"], "text_encoder_2": ["toy", "pixel"]} ``` diff --git a/docs/source/en/using-diffusers/write_own_pipeline.md b/docs/source/en/using-diffusers/write_own_pipeline.md index cc89edc80ec1..38fc9e6457dd 100644 --- a/docs/source/en/using-diffusers/write_own_pipeline.md +++ b/docs/source/en/using-diffusers/write_own_pipeline.md @@ -290,5 +290,5 @@ This is really what 🧨 Diffusers is designed for: to make it intuitive and eas For your next steps, feel free to: -* Learn how to [build and contribute a pipeline](contribute_pipeline) to 🧨 Diffusers. We can't wait and see what you'll come up with! +* Learn how to [build and contribute a pipeline](../using-diffusers/contribute_pipeline) to 🧨 Diffusers. We can't wait and see what you'll come up with! * Explore [existing pipelines](../api/pipelines/overview) in the library, and see if you can deconstruct and build a pipeline from scratch using the models and schedulers separately. From d1eb14bc357a179638be003cd61795f5dc2045f8 Mon Sep 17 00:00:00 2001 From: Steven Liu <59462357+stevhliu@users.noreply.github.com> Date: Wed, 1 Nov 2023 11:47:11 -0700 Subject: [PATCH 19/28] [docs] Lu lambdas (#5602) lu lambdas --- .../en/api/pipelines/stable_diffusion/overview.md | 12 ------------ .../stable_diffusion/stable_diffusion_xl.md | 3 +++ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/docs/source/en/api/pipelines/stable_diffusion/overview.md b/docs/source/en/api/pipelines/stable_diffusion/overview.md index 82b2597a7043..fe30e7177dbf 100644 --- a/docs/source/en/api/pipelines/stable_diffusion/overview.md +++ b/docs/source/en/api/pipelines/stable_diffusion/overview.md @@ -36,10 +36,8 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo Space - - @@ -49,7 +47,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionImg2Img @@ -58,7 +55,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionInpaint @@ -67,7 +63,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionDepth2Img @@ -76,7 +71,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionImageVariation @@ -85,7 +79,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionPipelineSafe @@ -94,7 +87,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusion2 @@ -103,7 +95,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionXL @@ -112,7 +103,6 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionLatentUpscale @@ -121,14 +111,12 @@ The table below summarizes the available Stable Diffusion pipelines, their suppo - StableDiffusionUpscale super-resolution - StableDiffusionLDM3D diff --git a/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_xl.md b/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_xl.md index aedb03d51caf..d257a6e91edc 100644 --- a/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_xl.md +++ b/docs/source/en/api/pipelines/stable_diffusion/stable_diffusion_xl.md @@ -20,6 +20,9 @@ The abstract from the paper is: ## Tips +- Using SDXL with a DPM++ scheduler for less than 50 steps is known to produce [visual artifacts](https://github.com/huggingface/diffusers/issues/5433) because the solver becomes numerically unstable. To fix this issue, take a look at this [PR](https://github.com/huggingface/diffusers/pull/5541) which recommends for ODE/SDE solvers: + - set `use_karras_sigmas=True` or `lu_lambdas=True` to improve image quality + - set `euler_at_final=True` if you're using a solver with uniform step sizes (DPM++2M or DPM++2M SDE) - Most SDXL checkpoints work best with an image size of 1024x1024. Image sizes of 768x768 and 512x512 are also supported, but the results aren't as good. Anything below 512x512 is not recommended and likely won't for for default checkpoints like [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0). - SDXL can pass a different prompt for each of the text encoders it was trained on. We can even pass different parts of the same prompt to the text encoders. - SDXL output images can be improved by making use of a refiner model in an image-to-image setting. From 151998e1c27d0e4432b3d2c488e1cfce4acfc8f3 Mon Sep 17 00:00:00 2001 From: clarencechen Date: Wed, 1 Nov 2023 13:22:56 -0700 Subject: [PATCH 20/28] Update final CPU offloading code for more diffusion pipelines (#5589) * Update final model offload for more pipelines Add test to ensure all pipeline components are returned to CPU after execution with model offloading * Add comment to explain early UNet offload in Text-to-Video pipeline * Style --- .../controlnet/pipeline_controlnet_inpaint_sd_xl.py | 5 ++--- .../controlnet/pipeline_controlnet_sd_xl_img2img.py | 5 ++--- .../stable_diffusion/pipeline_stable_diffusion_gligen.py | 5 ++--- .../pipeline_stable_diffusion_gligen_text_image.py | 5 ++--- .../stable_diffusion/pipeline_stable_diffusion_upscale.py | 5 ++--- .../pipelines/stable_diffusion/pipeline_stable_unclip.py | 5 ++--- .../stable_diffusion/pipeline_stable_unclip_img2img.py | 5 ++--- .../t2i_adapter/pipeline_stable_diffusion_xl_adapter.py | 5 ++--- .../pipeline_text_to_video_synth_img2img.py | 1 + tests/pipelines/test_pipelines_common.py | 8 ++++++++ 10 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py index 46c9f25b6eb6..f29d3bd51526 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -1604,9 +1604,8 @@ def denoising_value_valid(dnv): image = self.image_processor.postprocess(image, output_type=output_type) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image,) diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py index 5f9abb444f69..78c7c1cd8dbf 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py @@ -1433,9 +1433,8 @@ def __call__( image = self.image_processor.postprocess(image, output_type=output_type) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image,) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py index 90c38851681b..ef88230b4489 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen.py @@ -864,9 +864,8 @@ def __call__( image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image, has_nsfw_concept) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py index eef5fbef5809..c54114854aff 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_gligen_text_image.py @@ -1031,9 +1031,8 @@ def __call__( image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image, has_nsfw_concept) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py index 00ed46ffc6ad..da89505017cd 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py @@ -820,9 +820,8 @@ def __call__( if output_type == "pil" and self.watermarker is not None: image = self.watermarker.apply_watermark(image) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image, has_nsfw_concept) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py index 6539a4c62947..c81dd85f0e46 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py @@ -942,9 +942,8 @@ def __call__( image = self.image_processor.postprocess(image, output_type=output_type) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image,) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py index 4441e643e233..73638fdd15da 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py @@ -839,9 +839,8 @@ def __call__( image = self.image_processor.postprocess(image, output_type=output_type) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image,) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index 2a3fca7f4603..a5d745ee6994 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -1059,9 +1059,8 @@ def __call__( image = self.image_processor.postprocess(image, output_type=output_type) - # Offload last model to CPU - if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: - self.final_offload_hook.offload() + # Offload all models + self.maybe_free_model_hooks() if not return_dict: return (image,) diff --git a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py index 2f128aa448d6..45e0f5892d9d 100644 --- a/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +++ b/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py @@ -777,6 +777,7 @@ def __call__( if output_type == "latent": return TextToVideoSDPipelineOutput(frames=latents) + # manually for max memory savings if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: self.unet.to("cpu") diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index ae13d0d3e9fa..353add3b4d37 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -742,6 +742,14 @@ def test_model_cpu_offload_forward_pass(self, expected_max_diff=2e-4): max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() self.assertLess(max_diff, expected_max_diff, "CPU offloading should not affect the inference results") + self.assertTrue( + all( + v.device == "cpu" + for k, v in pipe.components.values() + if isinstance(v, torch.nn.Module) and k not in pipe._exclude_from_cpu_offload + ), + "CPU offloading should leave all pipeline components on the CPU after inference", + ) @unittest.skipIf( torch_device != "cuda" or not is_xformers_available(), From 5712c3d2eff84742f1abc86dc327bf1c966335c6 Mon Sep 17 00:00:00 2001 From: ilisparrow <4880273+ilisparrow@users.noreply.github.com> Date: Wed, 1 Nov 2023 21:25:38 +0100 Subject: [PATCH 21/28] [Core] enable lora for sdxl adapters too and add slow tests. (#5555) * Enable lora for sdxl adapters too. Issue #5516 * fix: assertion values. * Use numpy_cosine_similarity_distance on the arrays Co-authored-by: Dhruv Nair * Use numpy_cosine_similarity_distance on the arrays Co-authored-by: Dhruv Nair * Changed imports orders to pass tests Co-authored-by: Dhruv Nair --------- Co-authored-by: Ilias A Co-authored-by: Dhruv Nair Co-authored-by: Sayak Paul --- .../pipeline_stable_diffusion_xl_adapter.py | 75 +++++++++++++++++++ .../test_stable_diffusion_xl_adapter.py | 68 ++++++++++++++++- 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index a5d745ee6994..41ce6568ed18 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect +import os from typing import Any, Callable, Dict, List, Optional, Tuple, Union import numpy as np @@ -1066,3 +1067,77 @@ def __call__( return (image,) return StableDiffusionXLPipelineOutput(images=image) + + + # Overrride to properly handle the loading and unloading of the additional text encoder. + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.load_lora_weights + def load_lora_weights(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): + # We could have accessed the unet config from `lora_state_dict()` too. We pass + # it here explicitly to be able to tell that it's coming from an SDXL + # pipeline. + state_dict, network_alphas = self.lora_state_dict( + pretrained_model_name_or_path_or_dict, + unet_config=self.unet.config, + **kwargs, + ) + self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.unet) + + text_encoder_state_dict = {k: v for k, v in state_dict.items() if "text_encoder." in k} + if len(text_encoder_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder, + prefix="text_encoder", + lora_scale=self.lora_scale, + ) + + text_encoder_2_state_dict = {k: v for k, v in state_dict.items() if "text_encoder_2." in k} + if len(text_encoder_2_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_2_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder_2, + prefix="text_encoder_2", + lora_scale=self.lora_scale, + ) + + @classmethod + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.save_lora_weights + def save_lora_weights( + self, + save_directory: Union[str, os.PathLike], + unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + is_main_process: bool = True, + weight_name: str = None, + save_function: Callable = None, + safe_serialization: bool = True, + ): + state_dict = {} + + def pack_weights(layers, prefix): + layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers + layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()} + return layers_state_dict + + state_dict.update(pack_weights(unet_lora_layers, "unet")) + + if text_encoder_lora_layers and text_encoder_2_lora_layers: + state_dict.update(pack_weights(text_encoder_lora_layers, "text_encoder")) + state_dict.update(pack_weights(text_encoder_2_lora_layers, "text_encoder_2")) + + self.write_lora_layers( + state_dict=state_dict, + save_directory=save_directory, + is_main_process=is_main_process, + weight_name=weight_name, + save_function=save_function, + safe_serialization=safe_serialization, + ) + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._remove_text_encoder_monkey_patch + def _remove_text_encoder_monkey_patch(self): + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder) + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2) \ No newline at end of file diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py index 616aec6392f6..10ff9ff36901 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py @@ -14,6 +14,7 @@ # limitations under the License. import random +import gc import unittest import numpy as np @@ -29,10 +30,14 @@ StableDiffusionXLAdapterPipeline, T2IAdapter, UNet2DConditionModel, + EulerAncestralDiscreteScheduler, + ) from diffusers.utils import logging from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, torch_device - +from diffusers.utils import load_image +from diffusers.utils.torch_utils import randn_tensor +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS from ..test_pipelines_common import ( PipelineTesterMixin, @@ -560,3 +565,64 @@ def test_inference_batch_single_identical( if test_mean_pixel_difference: assert_mean_pixel_difference(output_batch[0][0], output[0][0]) + + +@slow +@require_torch_gpu +class AdapterSDXLPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_canny(self): + adapter = T2IAdapter.from_pretrained( + "TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16 + ).to("cpu") + pipe = StableDiffusionXLAdapterPipeline.from_pretrained( + 'stabilityai/stable-diffusion-xl-base-1.0', adapter=adapter, torch_dtype=torch.float16, variant="fp16", + ) + pipe.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors") + pipe.enable_sequential_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "toy" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + ) + + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images[0].shape == (768, 512, 3) + + original_image = images[0, -3:, -3:, -1].flatten() + assert numpy_cosine_similarity_distance(original_image, expected_image) < 1e-4 + assert np.allclose(original_image, expected_image, atol=1e-04) + + + def test_canny_lora(self): + adapter = T2IAdapter.from_pretrained( + "TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16 + ).to("cpu") + pipe = StableDiffusionXLAdapterPipeline.from_pretrained( + 'stabilityai/stable-diffusion-xl-base-1.0', adapter=adapter, torch_dtype=torch.float16, variant="fp16", + ) + pipe.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors") + pipe.enable_sequential_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "toy" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + ) + + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images[0].shape == (768, 512, 3) + + original_image = images[0, -3:, -3:, -1].flatten() + expected_image = np.array([0.50346327, 0.50708383, 0.50719553, 0.5135172, 0.5155377, 0.5066059, 0.49680984, 0.5005894, 0.48509413]) + assert numpy_cosine_similarity_distance(original_image, expected_image) < 1e-4 + From 839c2a5ece0af4e75530cb520d77bc7ed8acf474 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 1 Nov 2023 21:39:30 +0100 Subject: [PATCH 22/28] fix --- .../pipeline_stable_diffusion_xl_adapter.py | 3 +- .../test_stable_diffusion_xl_adapter.py | 63 +++++++------------ 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index 41ce6568ed18..c814e88096fa 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -1068,7 +1068,6 @@ def __call__( return StableDiffusionXLPipelineOutput(images=image) - # Overrride to properly handle the loading and unloading of the additional text encoder. # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.load_lora_weights def load_lora_weights(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): @@ -1140,4 +1139,4 @@ def pack_weights(layers, prefix): # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._remove_text_encoder_monkey_patch def _remove_text_encoder_monkey_patch(self): self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder) - self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2) \ No newline at end of file + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2) diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py index 10ff9ff36901..1c83e80f3d6f 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import random import gc +import random import unittest import numpy as np @@ -30,14 +30,17 @@ StableDiffusionXLAdapterPipeline, T2IAdapter, UNet2DConditionModel, - EulerAncestralDiscreteScheduler, - ) -from diffusers.utils import logging -from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, torch_device -from diffusers.utils import load_image -from diffusers.utils.torch_utils import randn_tensor -from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device +from diffusers.utils import load_image, logging +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS from ..test_pipelines_common import ( PipelineTesterMixin, @@ -575,38 +578,15 @@ def tearDown(self): gc.collect() torch.cuda.empty_cache() - def test_canny(self): - adapter = T2IAdapter.from_pretrained( - "TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16 - ).to("cpu") - pipe = StableDiffusionXLAdapterPipeline.from_pretrained( - 'stabilityai/stable-diffusion-xl-base-1.0', adapter=adapter, torch_dtype=torch.float16, variant="fp16", - ) - pipe.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors") - pipe.enable_sequential_cpu_offload() - pipe.set_progress_bar_config(disable=None) - - generator = torch.Generator(device="cpu").manual_seed(0) - prompt = "toy" - image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" - ) - - images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images - - assert images[0].shape == (768, 512, 3) - - original_image = images[0, -3:, -3:, -1].flatten() - assert numpy_cosine_similarity_distance(original_image, expected_image) < 1e-4 - assert np.allclose(original_image, expected_image, atol=1e-04) - - def test_canny_lora(self): - adapter = T2IAdapter.from_pretrained( - "TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16 - ).to("cpu") + adapter = T2IAdapter.from_pretrained("TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16).to( + "cpu" + ) pipe = StableDiffusionXLAdapterPipeline.from_pretrained( - 'stabilityai/stable-diffusion-xl-base-1.0', adapter=adapter, torch_dtype=torch.float16, variant="fp16", + "stabilityai/stable-diffusion-xl-base-1.0", + adapter=adapter, + torch_dtype=torch.float16, + variant="fp16", ) pipe.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors") pipe.enable_sequential_cpu_offload() @@ -615,7 +595,7 @@ def test_canny_lora(self): generator = torch.Generator(device="cpu").manual_seed(0) prompt = "toy" image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" ) images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images @@ -623,6 +603,7 @@ def test_canny_lora(self): assert images[0].shape == (768, 512, 3) original_image = images[0, -3:, -3:, -1].flatten() - expected_image = np.array([0.50346327, 0.50708383, 0.50719553, 0.5135172, 0.5155377, 0.5066059, 0.49680984, 0.5005894, 0.48509413]) + expected_image = np.array( + [0.50346327, 0.50708383, 0.50719553, 0.5135172, 0.5155377, 0.5066059, 0.49680984, 0.5005894, 0.48509413] + ) assert numpy_cosine_similarity_distance(original_image, expected_image) < 1e-4 - From 29cf163b95b7ebe4e6609d1e52c9ff226f4679c1 Mon Sep 17 00:00:00 2001 From: Chi Date: Thu, 2 Nov 2023 02:20:33 +0530 Subject: [PATCH 23/28] Remove Redundant Variables from Encoder and Decoder (#5569) * I added a new doc string to the class. This is more flexible to understanding other developers what are doing and where it's using. * Update src/diffusers/models/unet_2d_blocks.py This changes suggest by maintener. Co-authored-by: Sayak Paul * Update src/diffusers/models/unet_2d_blocks.py Add suggested text Co-authored-by: Sayak Paul * Update unet_2d_blocks.py I changed the Parameter to Args text. * Update unet_2d_blocks.py proper indentation set in this file. * Update unet_2d_blocks.py a little bit of change in the act_fun argument line. * I run the black command to reformat style in the code * Update unet_2d_blocks.py similar doc-string add to have in the original diffusion repository. * I removed the dummy variable defined in both the encoder and decoder. * Now, I run black package to reformat my file --------- Co-authored-by: Sayak Paul Co-authored-by: Dhruv Nair --- src/diffusers/models/vae.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/diffusers/models/vae.py b/src/diffusers/models/vae.py index da08bc360942..0f849a66eaea 100644 --- a/src/diffusers/models/vae.py +++ b/src/diffusers/models/vae.py @@ -130,9 +130,9 @@ def __init__( self.gradient_checkpointing = False - def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + def forward(self, sample: torch.FloatTensor) -> torch.FloatTensor: r"""The forward method of the `Encoder` class.""" - sample = x + sample = self.conv_in(sample) if self.training and self.gradient_checkpointing: @@ -273,9 +273,11 @@ def __init__( self.gradient_checkpointing = False - def forward(self, z: torch.FloatTensor, latent_embeds: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + def forward( + self, sample: torch.FloatTensor, latent_embeds: Optional[torch.FloatTensor] = None + ) -> torch.FloatTensor: r"""The forward method of the `Decoder` class.""" - sample = z + sample = self.conv_in(sample) upscale_dtype = next(iter(self.up_blocks.parameters())).dtype From 4f2bf673550a001ac5c8e474245f790675912829 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 1 Nov 2023 22:04:47 +0100 Subject: [PATCH 24/28] Revert "Fix the order of width and height of original size in SDXL training script" (#5614) Revert "Fix the order of width and height of original size in SDXL training script (#5382)" This reverts commit 45db049973df865cd63f4127057cc820842aadaf. --- examples/text_to_image/train_text_to_image_lora_sdxl.py | 2 +- examples/text_to_image/train_text_to_image_sdxl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/text_to_image/train_text_to_image_lora_sdxl.py b/examples/text_to_image/train_text_to_image_lora_sdxl.py index 74fc01aee3e6..6fbeae8b1f93 100644 --- a/examples/text_to_image/train_text_to_image_lora_sdxl.py +++ b/examples/text_to_image/train_text_to_image_lora_sdxl.py @@ -840,7 +840,7 @@ def preprocess_train(examples): all_images = [] crop_top_lefts = [] for image in images: - original_sizes.append((image.width, image.height)) + original_sizes.append((image.height, image.width)) image = train_resize(image) if args.center_crop: y1 = max(0, int(round((image.height - args.resolution) / 2.0))) diff --git a/examples/text_to_image/train_text_to_image_sdxl.py b/examples/text_to_image/train_text_to_image_sdxl.py index ea8ceff3952b..4a3048a0ba23 100644 --- a/examples/text_to_image/train_text_to_image_sdxl.py +++ b/examples/text_to_image/train_text_to_image_sdxl.py @@ -825,7 +825,7 @@ def preprocess_train(examples): all_images = [] crop_top_lefts = [] for image in images: - original_sizes.append((image.width, image.height)) + original_sizes.append((image.height, image.width)) image = train_resize(image) if args.center_crop: y1 = max(0, int(round((image.height - args.resolution) / 2.0))) From 02ba50c6104d40b745163fd14e84214b3db90112 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 1 Nov 2023 22:08:22 +0100 Subject: [PATCH 25/28] [`PEFT` / `LoRA`] Fix civitai bug when network alpha is an empty dict (#5608) * fix civitai bug * add test * up * fix test * added slow test. * style * Update src/diffusers/utils/peft_utils.py Co-authored-by: Benjamin Bossan * Update src/diffusers/utils/peft_utils.py --------- Co-authored-by: Benjamin Bossan --- src/diffusers/utils/peft_utils.py | 2 +- tests/lora/test_lora_layers_peft.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/diffusers/utils/peft_utils.py b/src/diffusers/utils/peft_utils.py index 940ad7fa14dc..158435a6e812 100644 --- a/src/diffusers/utils/peft_utils.py +++ b/src/diffusers/utils/peft_utils.py @@ -129,7 +129,7 @@ def get_peft_kwargs(rank_dict, network_alpha_dict, peft_state_dict, is_unet=True rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items())) rank_pattern = {k.split(".lora_B.")[0]: v for k, v in rank_pattern.items()} - if network_alpha_dict is not None: + if network_alpha_dict is not None and len(network_alpha_dict) > 0: if len(set(network_alpha_dict.values())) > 1: # get the alpha occuring the most number of times lora_alpha = collections.Counter(network_alpha_dict.values()).most_common()[0][0] diff --git a/tests/lora/test_lora_layers_peft.py b/tests/lora/test_lora_layers_peft.py index 0f61218e4f7f..6217d1cd28cd 100644 --- a/tests/lora/test_lora_layers_peft.py +++ b/tests/lora/test_lora_layers_peft.py @@ -22,6 +22,7 @@ import torch import torch.nn as nn import torch.nn.functional as F +from huggingface_hub import hf_hub_download from huggingface_hub.repocard import RepoCard from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer @@ -1772,6 +1773,28 @@ def test_sdxl_1_0_lora_with_sequential_cpu_offloading(self): self.assertTrue(np.allclose(images, expected, atol=1e-3)) release_memory(pipe) + def test_sd_load_civitai_empty_network_alpha(self): + """ + This test simply checks that loading a LoRA with an empty network alpha works fine + See: https://github.com/huggingface/diffusers/issues/5606 + """ + pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5").to("cuda") + pipeline.enable_sequential_cpu_offload() + civitai_path = hf_hub_download("ybelkada/test-ahi-civitai", "ahi_lora_weights.safetensors") + pipeline.load_lora_weights(civitai_path, adapter_name="ahri") + + images = pipeline( + "ahri, masterpiece, league of legends", + output_type="np", + generator=torch.manual_seed(156), + num_inference_steps=5, + ).images + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.0, 0.0, 0.0, 0.002557, 0.020954, 0.001792, 0.006581, 0.00591, 0.002995]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipeline) + def test_canny_lora(self): controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0") From b81c69e489aad3a0ba73798c459a33990dc4379c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Tolga=20Cang=C3=B6z?= <46008593+standardAI@users.noreply.github.com> Date: Thu, 2 Nov 2023 00:11:57 +0300 Subject: [PATCH 26/28] [Docs] Fix typos, improve, update at Get Started page (#5587) * Fix typos, improve, update * Update _toctree.yml * Update docs/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Update docs/README.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> * Apply Grammarly fixes --------- Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/README.md | 37 ++++++++++++++---------------- docs/source/en/_toctree.yml | 8 +++---- docs/source/en/index.md | 2 +- docs/source/en/installation.md | 8 +++++++ docs/source/en/quicktour.md | 32 +++++++++++++++----------- docs/source/en/stable_diffusion.md | 9 ++++---- 6 files changed, 54 insertions(+), 42 deletions(-) diff --git a/docs/README.md b/docs/README.md index fd0a3a58b0aa..30e5d430765e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,7 +71,7 @@ The `preview` command only works with existing doc files. When you add a complet Accepted files are Markdown (.md). Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting -the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/diffusers/blob/main/docs/source/_toctree.yml) file. +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/diffusers/blob/main/docs/source/en/_toctree.yml) file. ## Renaming section headers and moving sections @@ -81,14 +81,14 @@ Therefore, we simply keep a little map of moved sections at the end of the docum So if you renamed a section from: "Section A" to "Section B", then you can add at the end of the file: -``` +```md Sections that were moved: [ Section A ] ``` and of course, if you moved it to another file, then: -``` +```md Sections that were moved: [ Section A ] @@ -109,8 +109,8 @@ although we can write them directly in Markdown. Adding a new tutorial or section is done in two steps: -- Add a new file under `docs/source`. This file can either be ReStructuredText (.rst) or Markdown (.md). -- Link that file in `docs/source/_toctree.yml` on the correct toc-tree. +- Add a new Markdown (.md) file under `docs/source/`. +- Link that file in `docs/source//_toctree.yml` on the correct toc-tree. Make sure to put your new file under the proper section. It's unlikely to go in the first section (*Get Started*), so depending on the intended targets (beginners, more advanced users, or researchers) it should go in sections two, three, or four. @@ -119,7 +119,7 @@ depending on the intended targets (beginners, more advanced users, or researcher When adding a new pipeline: -- create a file `xxx.md` under `docs/source/api/pipelines` (don't hesitate to copy an existing file as template). +- Create a file `xxx.md` under `docs/source//api/pipelines` (don't hesitate to copy an existing file as template). - Link that file in (*Diffusers Summary*) section in `docs/source/api/pipelines/overview.md`, along with the link to the paper, and a colab notebook (if available). - Write a short overview of the diffusion model: - Overview with paper & authors @@ -128,9 +128,7 @@ When adding a new pipeline: - Possible an end-to-end example of how to use it - Add all the pipeline classes that should be linked in the diffusion model. These classes should be added using our Markdown syntax. By default as follows: -```py -## XXXPipeline - +``` [[autodoc]] XXXPipeline - all - __call__ @@ -138,7 +136,7 @@ When adding a new pipeline: This will include every public method of the pipeline that is documented, as well as the `__call__` method that is not documented by default. If you just want to add additional methods that are not documented, you can put the list of all methods to add in a list that contains `all`. -```py +``` [[autodoc]] XXXPipeline - all - __call__ @@ -148,7 +146,7 @@ This will include every public method of the pipeline that is documented, as wel - disable_xformers_memory_efficient_attention ``` -You can follow the same process to create a new scheduler under the `docs/source/api/schedulers` folder +You can follow the same process to create a new scheduler under the `docs/source//api/schedulers` folder. ### Writing source documentation @@ -164,7 +162,7 @@ provide its path. For instance: \[\`pipelines.ImagePipelineOutput\`\]. This will `pipelines.ImagePipelineOutput` in the description. To get rid of the path and only keep the name of the object you are linking to in the description, add a ~: \[\`~pipelines.ImagePipelineOutput\`\] will generate a link with `ImagePipelineOutput` in the description. -The same works for methods so you can either use \[\`XXXClass.method\`\] or \[~\`XXXClass.method\`\]. +The same works for methods so you can either use \[\`XXXClass.method\`\] or \[\`~XXXClass.method\`\]. #### Defining arguments in a method @@ -172,7 +170,7 @@ Arguments should be defined with the `Args:` (or `Arguments:` or `Parameters:`) an indentation. The argument should be followed by its type, with its shape if it is a tensor, a colon, and its description: -```py +``` Args: n_layers (`int`): The number of layers of the model. ``` @@ -182,7 +180,7 @@ after the argument. Here's an example showcasing everything so far: -```py +``` Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. @@ -197,16 +195,16 @@ For optional arguments or arguments with defaults we follow the following syntax following signature: ```py -def my_function(x: str = None, a: float = 1): +def my_function(x: str=None, a: float=3.14): ``` then its documentation should look like this: -```py +``` Args: x (`str`, *optional*): This argument controls ... - a (`float`, *optional*, defaults to 1): + a (`float`, *optional*, defaults to `3.14`): This argument is used to ... ``` @@ -235,14 +233,14 @@ building the return. Here's an example of a single value return: -```py +``` Returns: `List[int]`: A list of integers in the range [0, 1] --- 1 for a special token, 0 for a sequence token. ``` Here's an example of a tuple return, comprising several objects: -```py +``` Returns: `tuple(torch.FloatTensor)` comprising various elements depending on the configuration ([`BertConfig`]) and inputs: - ** loss** (*optional*, returned when `masked_lm_labels` is provided) `torch.FloatTensor` of shape `(1,)` -- @@ -268,4 +266,3 @@ We have an automatic script running with the `make style` command that will make This script may have some weird failures if you made a syntax mistake or if you uncover a bug. Therefore, it's recommended to commit your changes before running `make style`, so you can revert the changes done by that script easily. - diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index f2adb148cc28..0a983a3a9e47 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -12,7 +12,7 @@ - local: tutorials/tutorial_overview title: Overview - local: using-diffusers/write_own_pipeline - title: Understanding models and schedulers + title: Understanding pipelines, models and schedulers - local: tutorials/autopipeline title: AutoPipeline - local: tutorials/basic_training @@ -253,7 +253,7 @@ - local: api/pipelines/musicldm title: MusicLDM - local: api/pipelines/paint_by_example - title: PaintByExample + title: Paint By Example - local: api/pipelines/paradigms title: Parallel Sampling of Diffusion Models - local: api/pipelines/pix2pix_zero @@ -298,7 +298,7 @@ - local: api/pipelines/stable_diffusion/ldm3d_diffusion title: LDM3D Text-to-(RGB, Depth) - local: api/pipelines/stable_diffusion/adapter - title: Stable Diffusion T2I-adapter + title: Stable Diffusion T2I-Adapter - local: api/pipelines/stable_diffusion/gligen title: GLIGEN (Grounded Language-to-Image Generation) title: Stable Diffusion @@ -313,7 +313,7 @@ - local: api/pipelines/text_to_video_zero title: Text2Video-Zero - local: api/pipelines/unclip - title: UnCLIP + title: unCLIP - local: api/pipelines/latent_diffusion_uncond title: Unconditional Latent Diffusion - local: api/pipelines/unidiffuser diff --git a/docs/source/en/index.md b/docs/source/en/index.md index f4cf2e2114ec..ce6e79ee44d1 100644 --- a/docs/source/en/index.md +++ b/docs/source/en/index.md @@ -45,4 +45,4 @@ The library has three main components:

Technical descriptions of how 🤗 Diffusers classes and methods work.

-
\ No newline at end of file +
diff --git a/docs/source/en/installation.md b/docs/source/en/installation.md index ee15fb56384d..3bf1d46fd0c7 100644 --- a/docs/source/en/installation.md +++ b/docs/source/en/installation.md @@ -50,6 +50,14 @@ pip install diffusers["flax"] transformers +## Install with conda + +After activating your virtual environment, with `conda` (maintained by the community): + +```bash +conda install -c conda-forge diffusers +``` + ## Install from source Before installing 🤗 Diffusers from source, make sure you have PyTorch and 🤗 Accelerate installed. diff --git a/docs/source/en/quicktour.md b/docs/source/en/quicktour.md index 3cf6851e4683..c5ead9829cdc 100644 --- a/docs/source/en/quicktour.md +++ b/docs/source/en/quicktour.md @@ -26,7 +26,7 @@ The quicktour will show you how to use the [`DiffusionPipeline`] for inference, -The quicktour is a simplified version of the introductory 🧨 Diffusers [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/diffusers_intro.ipynb) to help you get started quickly. If you want to learn more about 🧨 Diffusers goal, design philosophy, and additional details about it's core API, check out the notebook! +The quicktour is a simplified version of the introductory 🧨 Diffusers [notebook](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/diffusers_intro.ipynb) to help you get started quickly. If you want to learn more about 🧨 Diffusers' goal, design philosophy, and additional details about its core API, check out the notebook! @@ -76,7 +76,7 @@ The [`DiffusionPipeline`] downloads and caches all modeling, tokenization, and s >>> pipeline StableDiffusionPipeline { "_class_name": "StableDiffusionPipeline", - "_diffusers_version": "0.13.1", + "_diffusers_version": "0.21.4", ..., "scheduler": [ "diffusers", @@ -133,7 +133,7 @@ Then load the saved weights into the pipeline: >>> pipeline = DiffusionPipeline.from_pretrained("./stable-diffusion-v1-5", use_safetensors=True) ``` -Now you can run the pipeline as you would in the section above. +Now, you can run the pipeline as you would in the section above. ### Swapping schedulers @@ -191,7 +191,7 @@ To use the model for inference, create the image shape with random Gaussian nois torch.Size([1, 3, 256, 256]) ``` -For inference, pass the noisy image to the model and a `timestep`. The `timestep` indicates how noisy the input image is, with more noise at the beginning and less at the end. This helps the model determine its position in the diffusion process, whether it is closer to the start or the end. Use the `sample` method to get the model output: +For inference, pass the noisy image and a `timestep` to the model. The `timestep` indicates how noisy the input image is, with more noise at the beginning and less at the end. This helps the model determine its position in the diffusion process, whether it is closer to the start or the end. Use the `sample` method to get the model output: ```py >>> with torch.no_grad(): @@ -210,23 +210,28 @@ Schedulers manage going from a noisy sample to a less noisy sample given the mod -For the quicktour, you'll instantiate the [`DDPMScheduler`] with it's [`~diffusers.ConfigMixin.from_config`] method: +For the quicktour, you'll instantiate the [`DDPMScheduler`] with its [`~diffusers.ConfigMixin.from_config`] method: ```py >>> from diffusers import DDPMScheduler ->>> scheduler = DDPMScheduler.from_config(repo_id) +>>> scheduler = DDPMScheduler.from_pretrained(repo_id) >>> scheduler DDPMScheduler { "_class_name": "DDPMScheduler", - "_diffusers_version": "0.13.1", + "_diffusers_version": "0.21.4", "beta_end": 0.02, "beta_schedule": "linear", "beta_start": 0.0001, "clip_sample": true, "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, "num_train_timesteps": 1000, "prediction_type": "epsilon", + "sample_max_value": 1.0, + "steps_offset": 0, + "thresholding": false, + "timestep_spacing": "leading", "trained_betas": null, "variance_type": "fixed_small" } @@ -234,13 +239,13 @@ DDPMScheduler { -💡 Notice how the scheduler is instantiated from a configuration. Unlike a model, a scheduler does not have trainable weights and is parameter-free! +💡 Unlike a model, a scheduler does not have trainable weights and is parameter-free! Some of the most important parameters are: -* `num_train_timesteps`: the length of the denoising process or in other words, the number of timesteps required to process random Gaussian noise into a data sample. +* `num_train_timesteps`: the length of the denoising process or, in other words, the number of timesteps required to process random Gaussian noise into a data sample. * `beta_schedule`: the type of noise schedule to use for inference and training. * `beta_start` and `beta_end`: the start and end noise values for the noise schedule. @@ -249,9 +254,10 @@ To predict a slightly less noisy image, pass the following to the scheduler's [` ```py >>> less_noisy_sample = scheduler.step(model_output=noisy_residual, timestep=2, sample=noisy_sample).prev_sample >>> less_noisy_sample.shape +torch.Size([1, 3, 256, 256]) ``` -The `less_noisy_sample` can be passed to the next `timestep` where it'll get even less noisier! Let's bring it all together now and visualize the entire denoising process. +The `less_noisy_sample` can be passed to the next `timestep` where it'll get even less noisy! Let's bring it all together now and visualize the entire denoising process. First, create a function that postprocesses and displays the denoised image as a `PIL.Image`: @@ -305,10 +311,10 @@ Sit back and watch as a cat is generated from nothing but noise! 😻 ## Next steps -Hopefully you generated some cool images with 🧨 Diffusers in this quicktour! For your next steps, you can: +Hopefully, you generated some cool images with 🧨 Diffusers in this quicktour! For your next steps, you can: * Train or finetune a model to generate your own images in the [training](./tutorials/basic_training) tutorial. * See example official and community [training or finetuning scripts](https://github.com/huggingface/diffusers/tree/main/examples#-diffusers-examples) for a variety of use cases. -* Learn more about loading, accessing, changing and comparing schedulers in the [Using different Schedulers](./using-diffusers/schedulers) guide. -* Explore prompt engineering, speed and memory optimizations, and tips and tricks for generating higher quality images with the [Stable Diffusion](./stable_diffusion) guide. +* Learn more about loading, accessing, changing, and comparing schedulers in the [Using different Schedulers](./using-diffusers/schedulers) guide. +* Explore prompt engineering, speed and memory optimizations, and tips and tricks for generating higher-quality images with the [Stable Diffusion](./stable_diffusion) guide. * Dive deeper into speeding up 🧨 Diffusers with guides on [optimized PyTorch on a GPU](./optimization/fp16), and inference guides for running [Stable Diffusion on Apple Silicon (M1/M2)](./optimization/mps) and [ONNX Runtime](./optimization/onnx). diff --git a/docs/source/en/stable_diffusion.md b/docs/source/en/stable_diffusion.md index f9407c3266c1..06eb5bf15f23 100644 --- a/docs/source/en/stable_diffusion.md +++ b/docs/source/en/stable_diffusion.md @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. Getting the [`DiffusionPipeline`] to generate images in a certain style or include what you want can be tricky. Often times, you have to run the [`DiffusionPipeline`] several times before you end up with an image you're happy with. But generating something out of nothing is a computationally intensive process, especially if you're running inference over and over again. -This is why it's important to get the most *computational* (speed) and *memory* (GPU RAM) efficiency from the pipeline to reduce the time between inference cycles so you can iterate faster. +This is why it's important to get the most *computational* (speed) and *memory* (GPU vRAM) efficiency from the pipeline to reduce the time between inference cycles so you can iterate faster. This tutorial walks you through how to generate faster and better with the [`DiffusionPipeline`]. @@ -108,6 +108,7 @@ pipeline.scheduler.compatibles diffusers.schedulers.scheduling_ddpm.DDPMScheduler, diffusers.schedulers.scheduling_dpmsolver_singlestep.DPMSolverSinglestepScheduler, diffusers.schedulers.scheduling_k_dpm_2_ancestral_discrete.KDPM2AncestralDiscreteScheduler, + diffusers.utils.dummy_torch_and_torchsde_objects.DPMSolverSDEScheduler, diffusers.schedulers.scheduling_heun_discrete.HeunDiscreteScheduler, diffusers.schedulers.scheduling_pndm.PNDMScheduler, diffusers.schedulers.scheduling_euler_ancestral_discrete.EulerAncestralDiscreteScheduler, @@ -115,7 +116,7 @@ pipeline.scheduler.compatibles ] ``` -The Stable Diffusion model uses the [`PNDMScheduler`] by default which usually requires ~50 inference steps, but more performant schedulers like [`DPMSolverMultistepScheduler`], require only ~20 or 25 inference steps. Use the [`ConfigMixin.from_config`] method to load a new scheduler: +The Stable Diffusion model uses the [`PNDMScheduler`] by default which usually requires ~50 inference steps, but more performant schedulers like [`DPMSolverMultistepScheduler`], require only ~20 or 25 inference steps. Use the [`~ConfigMixin.from_config`] method to load a new scheduler: ```python from diffusers import DPMSolverMultistepScheduler @@ -155,13 +156,13 @@ def get_inputs(batch_size=1): Start with `batch_size=4` and see how much memory you've consumed: ```python -from diffusers.utils import make_image_grid +from diffusers.utils import make_image_grid images = pipeline(**get_inputs(batch_size=4)).images make_image_grid(images, 2, 2) ``` -Unless you have a GPU with more RAM, the code above probably returned an `OOM` error! Most of the memory is taken up by the cross-attention layers. Instead of running this operation in a batch, you can run it sequentially to save a significant amount of memory. All you have to do is configure the pipeline to use the [`~DiffusionPipeline.enable_attention_slicing`] function: +Unless you have a GPU with more vRAM, the code above probably returned an `OOM` error! Most of the memory is taken up by the cross-attention layers. Instead of running this operation in a batch, you can run it sequentially to save a significant amount of memory. All you have to do is configure the pipeline to use the [`~DiffusionPipeline.enable_attention_slicing`] function: ```python pipeline.enable_attention_slicing() From c0f058265161178f2a88849e92b37ffdc81f1dcc Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 1 Nov 2023 22:18:58 +0100 Subject: [PATCH 27/28] [SDXL Adapter] Revert load lora (#5615) * fix * fix --- .../pipeline_stable_diffusion_xl_adapter.py | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index c814e88096fa..a5d745ee6994 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -13,7 +13,6 @@ # limitations under the License. import inspect -import os from typing import Any, Callable, Dict, List, Optional, Tuple, Union import numpy as np @@ -1067,76 +1066,3 @@ def __call__( return (image,) return StableDiffusionXLPipelineOutput(images=image) - - # Overrride to properly handle the loading and unloading of the additional text encoder. - # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.load_lora_weights - def load_lora_weights(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): - # We could have accessed the unet config from `lora_state_dict()` too. We pass - # it here explicitly to be able to tell that it's coming from an SDXL - # pipeline. - state_dict, network_alphas = self.lora_state_dict( - pretrained_model_name_or_path_or_dict, - unet_config=self.unet.config, - **kwargs, - ) - self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.unet) - - text_encoder_state_dict = {k: v for k, v in state_dict.items() if "text_encoder." in k} - if len(text_encoder_state_dict) > 0: - self.load_lora_into_text_encoder( - text_encoder_state_dict, - network_alphas=network_alphas, - text_encoder=self.text_encoder, - prefix="text_encoder", - lora_scale=self.lora_scale, - ) - - text_encoder_2_state_dict = {k: v for k, v in state_dict.items() if "text_encoder_2." in k} - if len(text_encoder_2_state_dict) > 0: - self.load_lora_into_text_encoder( - text_encoder_2_state_dict, - network_alphas=network_alphas, - text_encoder=self.text_encoder_2, - prefix="text_encoder_2", - lora_scale=self.lora_scale, - ) - - @classmethod - # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.save_lora_weights - def save_lora_weights( - self, - save_directory: Union[str, os.PathLike], - unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, - is_main_process: bool = True, - weight_name: str = None, - save_function: Callable = None, - safe_serialization: bool = True, - ): - state_dict = {} - - def pack_weights(layers, prefix): - layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers - layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()} - return layers_state_dict - - state_dict.update(pack_weights(unet_lora_layers, "unet")) - - if text_encoder_lora_layers and text_encoder_2_lora_layers: - state_dict.update(pack_weights(text_encoder_lora_layers, "text_encoder")) - state_dict.update(pack_weights(text_encoder_2_lora_layers, "text_encoder_2")) - - self.write_lora_layers( - state_dict=state_dict, - save_directory=save_directory, - is_main_process=is_main_process, - weight_name=weight_name, - save_function=save_function, - safe_serialization=safe_serialization, - ) - - # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._remove_text_encoder_monkey_patch - def _remove_text_encoder_monkey_patch(self): - self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder) - self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2) From 75ea54a1512ac443d517ab35cb9bf45f8d6f326e Mon Sep 17 00:00:00 2001 From: Steven Liu <59462357+stevhliu@users.noreply.github.com> Date: Wed, 1 Nov 2023 15:36:22 -0700 Subject: [PATCH 28/28] [docs] Kandinsky guide (#4555) * kandinsky 2.1 first draft * add kandinsky 2.2 * fix identical section headers * try hfoptions syntax * add img2img * add inpaint * add interpolate * fix tag * more cleanups * typo * update hfoptions id * align hfoptions tags --- docs/source/en/_toctree.yml | 6 +- docs/source/en/api/pipelines/kandinsky.md | 436 +---------- docs/source/en/api/pipelines/kandinsky_v22.md | 323 +------- docs/source/en/using-diffusers/kandinsky.md | 692 ++++++++++++++++++ 4 files changed, 740 insertions(+), 717 deletions(-) create mode 100644 docs/source/en/using-diffusers/kandinsky.md diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 0a983a3a9e47..488219e7a88e 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -40,6 +40,8 @@ title: Push files to the Hub title: Loading & Hub - sections: + - local: using-diffusers/pipeline_overview + title: Overview - local: using-diffusers/unconditional_image_generation title: Unconditional image generation - local: using-diffusers/conditional_image_generation @@ -70,6 +72,8 @@ title: Overview - local: using-diffusers/sdxl title: Stable Diffusion XL + - local: using-diffusers/kandinsky + title: Kandinsky - local: using-diffusers/controlnet title: ControlNet - local: using-diffusers/shap-e @@ -241,7 +245,7 @@ - local: api/pipelines/pix2pix title: InstructPix2Pix - local: api/pipelines/kandinsky - title: Kandinsky + title: Kandinsky 2.1 - local: api/pipelines/kandinsky_v22 title: Kandinsky 2.2 - local: api/pipelines/latent_consistency_models diff --git a/docs/source/en/api/pipelines/kandinsky.md b/docs/source/en/api/pipelines/kandinsky.md index 086821a3bc0a..30bc29a5e12e 100644 --- a/docs/source/en/api/pipelines/kandinsky.md +++ b/docs/source/en/api/pipelines/kandinsky.md @@ -7,462 +7,60 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# Kandinsky +# Kandinsky 2.1 -## Overview +Kandinsky 2.1 is created by [Arseniy Shakhmatov](https://github.com/cene555), [Anton Razzhigaev](https://github.com/razzant), [Aleksandr Nikolich](https://github.com/AlexWortega), [Igor Pavlov](https://github.com/boomb0om), [Andrey Kuznetsov](https://github.com/kuznetsoffandrey) and [Denis Dimitrov](https://github.com/denndimitrov). -Kandinsky inherits best practices from [DALL-E 2](https://huggingface.co/papers/2204.06125) and [Latent Diffusion](https://huggingface.co/docs/diffusers/api/pipelines/latent_diffusion), while introducing some new ideas. +The description from it's GitHub page is: -It uses [CLIP](https://huggingface.co/docs/transformers/model_doc/clip) for encoding images and text, and a diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach enhances the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation. +*Kandinsky 2.1 inherits best practicies from Dall-E 2 and Latent diffusion, while introducing some new ideas. As text and image encoder it uses CLIP model and diffusion image prior (mapping) between latent spaces of CLIP modalities. This approach increases the visual performance of the model and unveils new horizons in blending images and text-guided image manipulation.* -The Kandinsky model is created by [Arseniy Shakhmatov](https://github.com/cene555), [Anton Razzhigaev](https://github.com/razzant), [Aleksandr Nikolich](https://github.com/AlexWortega), [Igor Pavlov](https://github.com/boomb0om), [Andrey Kuznetsov](https://github.com/kuznetsoffandrey) and [Denis Dimitrov](https://github.com/denndimitrov). The original codebase can be found [here](https://github.com/ai-forever/Kandinsky-2) - - -## Usage example - -In the following, we will walk you through some examples of how to use the Kandinsky pipelines to create some visually aesthetic artwork. - -### Text-to-Image Generation - -For text-to-image generation, we need to use both [`KandinskyPriorPipeline`] and [`KandinskyPipeline`]. -The first step is to encode text prompts with CLIP and then diffuse the CLIP text embeddings to CLIP image embeddings, -as first proposed in [DALL-E 2](https://cdn.openai.com/papers/dall-e-2.pdf). -Let's throw a fun prompt at Kandinsky to see what it comes up with. - -```py -prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" -``` - -First, let's instantiate the prior pipeline and the text-to-image pipeline. Both -pipelines are diffusion models. - - -```py -from diffusers import DiffusionPipeline -import torch - -pipe_prior = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16) -pipe_prior.to("cuda") - -t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) -t2i_pipe.to("cuda") -``` - - - -By default, the text-to-image pipeline use [`DDIMScheduler`], you can change the scheduler to [`DDPMScheduler`] - -```py -scheduler = DDPMScheduler.from_pretrained("kandinsky-community/kandinsky-2-1", subfolder="ddpm_scheduler") -t2i_pipe = DiffusionPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-1", scheduler=scheduler, torch_dtype=torch.float16 -) -t2i_pipe.to("cuda") -``` - - - -Now we pass the prompt through the prior to generate image embeddings. The prior -returns both the image embeddings corresponding to the prompt and negative/unconditional image -embeddings corresponding to an empty string. - -```py -image_embeds, negative_image_embeds = pipe_prior(prompt, guidance_scale=1.0).to_tuple() -``` - - - -The text-to-image pipeline expects both `image_embeds`, `negative_image_embeds` and the original -`prompt` as the text-to-image pipeline uses another text encoder to better guide the second diffusion -process of `t2i_pipe`. - -By default, the prior returns unconditioned negative image embeddings corresponding to the negative prompt of `""`. -For better results, you can also pass a `negative_prompt` to the prior. This will increase the effective batch size -of the prior by a factor of 2. - -```py -prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" -negative_prompt = "low quality, bad quality" - -image_embeds, negative_image_embeds = pipe_prior(prompt, negative_prompt, guidance_scale=1.0).to_tuple() -``` - - - - -Next, we can pass the embeddings as well as the prompt to the text-to-image pipeline. Remember that -in case you are using a customized negative prompt, that you should pass this one also to the text-to-image pipelines -with `negative_prompt=negative_prompt`: - -```py -image = t2i_pipe( - prompt, image_embeds=image_embeds, negative_image_embeds=negative_image_embeds, height=768, width=768 -).images[0] -image.save("cheeseburger_monster.png") -``` - -One cheeseburger monster coming up! Enjoy! - -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/cheeseburger.png) - - - -We also provide an end-to-end Kandinsky pipeline [`KandinskyCombinedPipeline`], which combines both the prior pipeline and text-to-image pipeline, and lets you perform inference in a single step. You can create the combined pipeline with the [`~AutoPipelineForText2Image.from_pretrained`] method - -```python -from diffusers import AutoPipelineForText2Image -import torch - -pipe = AutoPipelineForText2Image.from_pretrained( - "kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16 -) -pipe.enable_model_cpu_offload() -``` - -Under the hood, it will automatically load both [`KandinskyPriorPipeline`] and [`KandinskyPipeline`]. To generate images, you no longer need to call both pipelines and pass the outputs from one to another. You only need to call the combined pipeline once. You can set different `guidance_scale` and `num_inference_steps` for the prior pipeline with the `prior_guidance_scale` and `prior_num_inference_steps` arguments. - -```python -prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" -negative_prompt = "low quality, bad quality" - -image = pipe(prompt=prompt, negative_prompt=negative_prompt, prior_guidance_scale =1.0, guidance_scacle = 4.0, height=768, width=768).images[0] -``` - - -The Kandinsky model works extremely well with creative prompts. Here is some of the amazing art that can be created using the exact same process but with different prompts. - -```python -prompt = "bird eye view shot of a full body woman with cyan light orange magenta makeup, digital art, long braided hair her face separated by makeup in the style of yin Yang surrealism, symmetrical face, real image, contrasting tone, pastel gradient background" -``` -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/hair.png) - -```python -prompt = "A car exploding into colorful dust" -``` -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/dusts.png) - -```python -prompt = "editorial photography of an organic, almost liquid smoke style armchair" -``` -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/smokechair.png) - -```python -prompt = "birds eye view of a quilted paper style alien planet landscape, vibrant colours, Cinematic lighting" -``` -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/alienplanet.png) - - - -### Text Guided Image-to-Image Generation - -The same Kandinsky model weights can be used for text-guided image-to-image translation. In this case, just make sure to load the weights using the [`KandinskyImg2ImgPipeline`] pipeline. - -**Note**: You can also directly move the weights of the text-to-image pipelines to the image-to-image pipelines -without loading them twice by making use of the [`~DiffusionPipeline.components`] function as explained [here](#converting-between-different-pipelines). - -Let's download an image. - -```python -from PIL import Image -import requests -from io import BytesIO - -# download image -url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" -response = requests.get(url) -original_image = Image.open(BytesIO(response.content)).convert("RGB") -original_image = original_image.resize((768, 512)) -``` - -![img](https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg) - -```python -import torch -from diffusers import KandinskyImg2ImgPipeline, KandinskyPriorPipeline - -# create prior -pipe_prior = KandinskyPriorPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 -) -pipe_prior.to("cuda") - -# create img2img pipeline -pipe = KandinskyImg2ImgPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) -pipe.to("cuda") - -prompt = "A fantasy landscape, Cinematic lighting" -negative_prompt = "low quality, bad quality" - -image_embeds, negative_image_embeds = pipe_prior(prompt, negative_prompt).to_tuple() - -out = pipe( - prompt, - image=original_image, - image_embeds=image_embeds, - negative_image_embeds=negative_image_embeds, - height=768, - width=768, - strength=0.3, -) - -out.images[0].save("fantasy_land.png") -``` - -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/img2img_fantasyland.png) - - - - -You can also use the [`KandinskyImg2ImgCombinedPipeline`] for end-to-end image-to-image generation with Kandinsky 2.1 - -```python -from diffusers import AutoPipelineForImage2Image -import torch -import requests -from io import BytesIO -from PIL import Image -import os - -pipe = AutoPipelineForImage2Image.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) -pipe.enable_model_cpu_offload() - -prompt = "A fantasy landscape, Cinematic lighting" -negative_prompt = "low quality, bad quality" - -url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" - -response = requests.get(url) -original_image = Image.open(BytesIO(response.content)).convert("RGB") -original_image.thumbnail((768, 768)) - -image = pipe(prompt=prompt, image=original_image, strength=0.3).images[0] -``` - - -### Text Guided Inpainting Generation - -You can use [`KandinskyInpaintPipeline`] to edit images. In this example, we will add a hat to the portrait of a cat. - -```py -from diffusers import KandinskyInpaintPipeline, KandinskyPriorPipeline -from diffusers.utils import load_image -import torch -import numpy as np - -pipe_prior = KandinskyPriorPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 -) -pipe_prior.to("cuda") - -prompt = "a hat" -prior_output = pipe_prior(prompt) - -pipe = KandinskyInpaintPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16) -pipe.to("cuda") - -init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" -) - -mask = np.zeros((768, 768), dtype=np.float32) -# Let's mask out an area above the cat's head -mask[:250, 250:-250] = 1 - -out = pipe( - prompt, - image=init_image, - mask_image=mask, - **prior_output, - height=768, - width=768, - num_inference_steps=150, -) - -image = out.images[0] -image.save("cat_with_hat.png") -``` -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/inpaint_cat_hat.png) - - - -To use the [`KandinskyInpaintCombinedPipeline`] to perform end-to-end image inpainting generation, you can run below code instead - -```python -from diffusers import AutoPipelineForInpainting - -pipe = AutoPipelineForInpainting.from_pretrained("kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16) -pipe.enable_model_cpu_offload() -image = pipe(prompt=prompt, image=original_image, mask_image=mask).images[0] -``` - - -🚨🚨🚨 __Breaking change for Kandinsky Mask Inpainting__ 🚨🚨🚨 - -We introduced a breaking change for Kandinsky inpainting pipeline in the following pull request: https://github.com/huggingface/diffusers/pull/4207. Previously we accepted a mask format where black pixels represent the masked-out area. This is inconsistent with all other pipelines in diffusers. We have changed the mask format in Knaindsky and now using white pixels instead. -Please upgrade your inpainting code to follow the above. If you are using Kandinsky Inpaint in production. You now need to change the mask to: - -```python -# For PIL input -import PIL.ImageOps -mask = PIL.ImageOps.invert(mask) - -# For PyTorch and Numpy input -mask = 1 - mask -``` - -### Interpolate - -The [`KandinskyPriorPipeline`] also comes with a cool utility function that will allow you to interpolate the latent space of different images and texts super easily. Here is an example of how you can create an Impressionist-style portrait for your pet based on "The Starry Night". - -Note that you can interpolate between texts and images - in the below example, we passed a text prompt "a cat" and two images to the `interplate` function, along with a `weights` variable containing the corresponding weights for each condition we interplate. - -```python -from diffusers import KandinskyPriorPipeline, KandinskyPipeline -from diffusers.utils import load_image -import PIL - -import torch - -pipe_prior = KandinskyPriorPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 -) -pipe_prior.to("cuda") - -img1 = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" -) - -img2 = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/starry_night.jpeg" -) - -# add all the conditions we want to interpolate, can be either text or image -images_texts = ["a cat", img1, img2] - -# specify the weights for each condition in images_texts -weights = [0.3, 0.3, 0.4] - -# We can leave the prompt empty -prompt = "" -prior_out = pipe_prior.interpolate(images_texts, weights) - -pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) -pipe.to("cuda") - -image = pipe(prompt, **prior_out, height=768, width=768).images[0] - -image.save("starry_cat.png") -``` -![img](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/kandinsky-docs/starry_cat.png) - -## Optimization - -Running Kandinsky in inference requires running both a first prior pipeline: [`KandinskyPriorPipeline`] -and a second image decoding pipeline which is one of [`KandinskyPipeline`], [`KandinskyImg2ImgPipeline`], or [`KandinskyInpaintPipeline`]. - -The bulk of the computation time will always be the second image decoding pipeline, so when looking -into optimizing the model, one should look into the second image decoding pipeline. - -When running with PyTorch < 2.0, we strongly recommend making use of [`xformers`](https://github.com/facebookresearch/xformers) -to speed-up the optimization. This can be done by simply running: - -```py -from diffusers import DiffusionPipeline -import torch - -t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) -t2i_pipe.enable_xformers_memory_efficient_attention() -``` - -When running on PyTorch >= 2.0, PyTorch's SDPA attention will automatically be used. For more information on -PyTorch's SDPA, feel free to have a look at [this blog post](https://pytorch.org/blog/accelerated-diffusers-pt-20/). - -To have explicit control , you can also manually set the pipeline to use PyTorch's 2.0 efficient attention: - -```py -from diffusers.models.attention_processor import AttnAddedKVProcessor2_0 - -t2i_pipe.unet.set_attn_processor(AttnAddedKVProcessor2_0()) -``` - -The slowest and most memory intense attention processor is the default `AttnAddedKVProcessor` processor. -We do **not** recommend using it except for testing purposes or cases where very high determistic behaviour is desired. -You can set it with: - -```py -from diffusers.models.attention_processor import AttnAddedKVProcessor - -t2i_pipe.unet.set_attn_processor(AttnAddedKVProcessor()) -``` - -With PyTorch >= 2.0, you can also use Kandinsky with `torch.compile` which depending -on your hardware can significantly speed-up your inference time once the model is compiled. -To use Kandinsksy with `torch.compile`, you can do: - -```py -t2i_pipe.unet.to(memory_format=torch.channels_last) -t2i_pipe.unet = torch.compile(t2i_pipe.unet, mode="reduce-overhead", fullgraph=True) -``` - -After compilation you should see a very fast inference time. For more information, -feel free to have a look at [Our PyTorch 2.0 benchmark](https://huggingface.co/docs/diffusers/main/en/optimization/torch2.0). +The original codebase can be found at [ai-forever/Kandinsky-2](https://github.com/ai-forever/Kandinsky-2). -To generate images directly from a single pipeline, you can use [`KandinskyCombinedPipeline`], [`KandinskyImg2ImgCombinedPipeline`], [`KandinskyInpaintCombinedPipeline`]. -These combined pipelines wrap the [`KandinskyPriorPipeline`] and [`KandinskyPipeline`], [`KandinskyImg2ImgPipeline`], [`KandinskyInpaintPipeline`] respectively into a single -pipeline for a simpler user experience +Check out the [Kandinsky Community](https://huggingface.co/kandinsky-community) organization on the Hub for the official model checkpoints for tasks like text-to-image, image-to-image, and inpainting. -## Available Pipelines: - -| Pipeline | Tasks | -|---|---| -| [pipeline_kandinsky.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py) | *Text-to-Image Generation* | -| [pipeline_kandinsky_combined.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky_combined.py) | *End-to-end Text-to-Image, image-to-image, Inpainting Generation* | -| [pipeline_kandinsky_inpaint.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py) | *Image-Guided Image Generation* | -| [pipeline_kandinsky_img2img.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py) | *Image-Guided Image Generation* | - - -### KandinskyPriorPipeline +## KandinskyPriorPipeline [[autodoc]] KandinskyPriorPipeline - all - __call__ - interpolate -### KandinskyPipeline +## KandinskyPipeline [[autodoc]] KandinskyPipeline - all - __call__ -### KandinskyImg2ImgPipeline +## KandinskyCombinedPipeline -[[autodoc]] KandinskyImg2ImgPipeline +[[autodoc]] KandinskyCombinedPipeline - all - __call__ -### KandinskyInpaintPipeline +## KandinskyImg2ImgPipeline -[[autodoc]] KandinskyInpaintPipeline +[[autodoc]] KandinskyImg2ImgPipeline - all - __call__ -### KandinskyCombinedPipeline +## KandinskyImg2ImgCombinedPipeline -[[autodoc]] KandinskyCombinedPipeline +[[autodoc]] KandinskyImg2ImgCombinedPipeline - all - __call__ -### KandinskyImg2ImgCombinedPipeline +## KandinskyInpaintPipeline -[[autodoc]] KandinskyImg2ImgCombinedPipeline +[[autodoc]] KandinskyInpaintPipeline - all - __call__ -### KandinskyInpaintCombinedPipeline +## KandinskyInpaintCombinedPipeline [[autodoc]] KandinskyInpaintCombinedPipeline - all diff --git a/docs/source/en/api/pipelines/kandinsky_v22.md b/docs/source/en/api/pipelines/kandinsky_v22.md index 44c10fd07789..350b96c3a9be 100644 --- a/docs/source/en/api/pipelines/kandinsky_v22.md +++ b/docs/source/en/api/pipelines/kandinsky_v22.md @@ -9,348 +9,77 @@ specific language governing permissions and limitations under the License. # Kandinsky 2.2 -The Kandinsky 2.2 release includes robust new text-to-image models that support text-to-image generation, image-to-image generation, image interpolation, and text-guided image inpainting. The general workflow to perform these tasks using Kandinsky 2.2 is the same as in Kandinsky 2.1. First, you will need to use a prior pipeline to generate image embeddings based on your text prompt, and then use one of the image decoding pipelines to generate the output image. The only difference is that in Kandinsky 2.2, all of the decoding pipelines no longer accept the `prompt` input, and the image generation process is conditioned with only `image_embeds` and `negative_image_embeds`. +Kandinsky 2.1 is created by [Arseniy Shakhmatov](https://github.com/cene555), [Anton Razzhigaev](https://github.com/razzant), [Aleksandr Nikolich](https://github.com/AlexWortega), [Igor Pavlov](https://github.com/boomb0om), [Andrey Kuznetsov](https://github.com/kuznetsoffandrey) and [Denis Dimitrov](https://github.com/denndimitrov). -Same as with Kandinsky 2.1, the easiest way to perform text-to-image generation is to use the combined Kandinsky pipeline. This process is exactly the same as Kandinsky 2.1. All you need to do is to replace the Kandinsky 2.1 checkpoint with 2.2. +The description from it's GitHub page is: -```python -from diffusers import AutoPipelineForText2Image -import torch +*Kandinsky 2.2 brings substantial improvements upon its predecessor, Kandinsky 2.1, by introducing a new, more powerful image encoder - CLIP-ViT-G and the ControlNet support. The switch to CLIP-ViT-G as the image encoder significantly increases the model's capability to generate more aesthetic pictures and better understand text, thus enhancing the model's overall performance. The addition of the ControlNet mechanism allows the model to effectively control the process of generating images. This leads to more accurate and visually appealing outputs and opens new possibilities for text-guided image manipulation.* -pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16) -pipe.enable_model_cpu_offload() - -prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" -negative_prompt = "low quality, bad quality" - -image = pipe(prompt=prompt, negative_prompt=negative_prompt, prior_guidance_scale =1.0, height=768, width=768).images[0] -``` - -Now, let's look at an example where we take separate steps to run the prior pipeline and text-to-image pipeline. This way, we can understand what's happening under the hood and how Kandinsky 2.2 differs from Kandinsky 2.1. - -First, let's create the prior pipeline and text-to-image pipeline with Kandinsky 2.2 checkpoints. - -```python -from diffusers import DiffusionPipeline -import torch - -pipe_prior = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16) -pipe_prior.to("cuda") - -t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16) -t2i_pipe.to("cuda") -``` - -You can then use `pipe_prior` to generate image embeddings. - -```python -prompt = "portrait of a women, blue eyes, cinematic" -negative_prompt = "low quality, bad quality" - -image_embeds, negative_image_embeds = pipe_prior(prompt, guidance_scale=1.0).to_tuple() -``` - -Now you can pass these embeddings to the text-to-image pipeline. When using Kandinsky 2.2 you don't need to pass the `prompt` (but you do with the previous version, Kandinsky 2.1). - -``` -image = t2i_pipe(image_embeds=image_embeds, negative_image_embeds=negative_image_embeds, height=768, width=768).images[ - 0 -] -image.save("portrait.png") -``` -![img](https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinskyv22/%20blue%20eyes.png) - -We used the text-to-image pipeline as an example, but the same process applies to all decoding pipelines in Kandinsky 2.2. For more information, please refer to our API section for each pipeline. - -### Text-to-Image Generation with ControlNet Conditioning - -In the following, we give a simple example of how to use [`KandinskyV22ControlnetPipeline`] to add control to the text-to-image generation with a depth image. - -First, let's take an image and extract its depth map. - -```python -from diffusers.utils import load_image - -img = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinskyv22/cat.png" -).resize((768, 768)) -``` -![img](https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinskyv22/cat.png) - -We can use the `depth-estimation` pipeline from transformers to process the image and retrieve its depth map. - -```python -import torch -import numpy as np - -from transformers import pipeline -from diffusers.utils import load_image - - -def make_hint(image, depth_estimator): - image = depth_estimator(image)["depth"] - image = np.array(image) - image = image[:, :, None] - image = np.concatenate([image, image, image], axis=2) - detected_map = torch.from_numpy(image).float() / 255.0 - hint = detected_map.permute(2, 0, 1) - return hint - - -depth_estimator = pipeline("depth-estimation") -hint = make_hint(img, depth_estimator).unsqueeze(0).half().to("cuda") -``` -Now, we load the prior pipeline and the text-to-image controlnet pipeline - -```python -from diffusers import KandinskyV22PriorPipeline, KandinskyV22ControlnetPipeline - -pipe_prior = KandinskyV22PriorPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 -) -pipe_prior = pipe_prior.to("cuda") - -pipe = KandinskyV22ControlnetPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 -) -pipe = pipe.to("cuda") -``` - -We pass the prompt and negative prompt through the prior to generate image embeddings - -```python -prompt = "A robot, 4k photo" - -negative_prior_prompt = "lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature" - -generator = torch.Generator(device="cuda").manual_seed(43) -image_emb, zero_image_emb = pipe_prior( - prompt=prompt, negative_prompt=negative_prior_prompt, generator=generator -).to_tuple() -``` - -Now we can pass the image embeddings and the depth image we extracted to the controlnet pipeline. With Kandinsky 2.2, only prior pipelines accept `prompt` input. You do not need to pass the prompt to the controlnet pipeline. - -```python -images = pipe( - image_embeds=image_emb, - negative_image_embeds=zero_image_emb, - hint=hint, - num_inference_steps=50, - generator=generator, - height=768, - width=768, -).images - -images[0].save("robot_cat.png") -``` - -The output image looks as follow: -![img](https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinskyv22/robot_cat_text2img.png) - -### Image-to-Image Generation with ControlNet Conditioning - -Kandinsky 2.2 also includes a [`KandinskyV22ControlnetImg2ImgPipeline`] that will allow you to add control to the image generation process with both the image and its depth map. This pipeline works really well with [`KandinskyV22PriorEmb2EmbPipeline`], which generates image embeddings based on both a text prompt and an image. - -For our robot cat example, we will pass the prompt and cat image together to the prior pipeline to generate an image embedding. We will then use that image embedding and the depth map of the cat to further control the image generation process. - -We can use the same cat image and its depth map from the last example. - -```python -import torch -import numpy as np - -from diffusers import KandinskyV22PriorEmb2EmbPipeline, KandinskyV22ControlnetImg2ImgPipeline -from diffusers.utils import load_image -from transformers import pipeline - -img = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinskyv22/cat.png" -).resize((768, 768)) - - -def make_hint(image, depth_estimator): - image = depth_estimator(image)["depth"] - image = np.array(image) - image = image[:, :, None] - image = np.concatenate([image, image, image], axis=2) - detected_map = torch.from_numpy(image).float() / 255.0 - hint = detected_map.permute(2, 0, 1) - return hint - - -depth_estimator = pipeline("depth-estimation") -hint = make_hint(img, depth_estimator).unsqueeze(0).half().to("cuda") - -pipe_prior = KandinskyV22PriorEmb2EmbPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 -) -pipe_prior = pipe_prior.to("cuda") - -pipe = KandinskyV22ControlnetImg2ImgPipeline.from_pretrained( - "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 -) -pipe = pipe.to("cuda") - -prompt = "A robot, 4k photo" -negative_prior_prompt = "lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature" - -generator = torch.Generator(device="cuda").manual_seed(43) - -# run prior pipeline - -img_emb = pipe_prior(prompt=prompt, image=img, strength=0.85, generator=generator) -negative_emb = pipe_prior(prompt=negative_prior_prompt, image=img, strength=1, generator=generator) - -# run controlnet img2img pipeline -images = pipe( - image=img, - strength=0.5, - image_embeds=img_emb.image_embeds, - negative_image_embeds=negative_emb.image_embeds, - hint=hint, - num_inference_steps=50, - generator=generator, - height=768, - width=768, -).images - -images[0].save("robot_cat.png") -``` - -Here is the output. Compared with the output from our text-to-image controlnet example, it kept a lot more cat facial details from the original image and worked into the robot style we asked for. - -![img](https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinskyv22/robot_cat.png) - -## Optimization - -Running Kandinsky in inference requires running both a first prior pipeline: [`KandinskyPriorPipeline`] -and a second image decoding pipeline which is one of [`KandinskyPipeline`], [`KandinskyImg2ImgPipeline`], or [`KandinskyInpaintPipeline`]. - -The bulk of the computation time will always be the second image decoding pipeline, so when looking -into optimizing the model, one should look into the second image decoding pipeline. - -When running with PyTorch < 2.0, we strongly recommend making use of [`xformers`](https://github.com/facebookresearch/xformers) -to speed-up the optimization. This can be done by simply running: - -```py -from diffusers import DiffusionPipeline -import torch - -t2i_pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16) -t2i_pipe.enable_xformers_memory_efficient_attention() -``` - -When running on PyTorch >= 2.0, PyTorch's SDPA attention will automatically be used. For more information on -PyTorch's SDPA, feel free to have a look at [this blog post](https://pytorch.org/blog/accelerated-diffusers-pt-20/). - -To have explicit control , you can also manually set the pipeline to use PyTorch's 2.0 efficient attention: - -```py -from diffusers.models.attention_processor import AttnAddedKVProcessor2_0 - -t2i_pipe.unet.set_attn_processor(AttnAddedKVProcessor2_0()) -``` - -The slowest and most memory intense attention processor is the default `AttnAddedKVProcessor` processor. -We do **not** recommend using it except for testing purposes or cases where very high determistic behaviour is desired. -You can set it with: - -```py -from diffusers.models.attention_processor import AttnAddedKVProcessor - -t2i_pipe.unet.set_attn_processor(AttnAddedKVProcessor()) -``` - -With PyTorch >= 2.0, you can also use Kandinsky with `torch.compile` which depending -on your hardware can significantly speed-up your inference time once the model is compiled. -To use Kandinsksy with `torch.compile`, you can do: - -```py -t2i_pipe.unet.to(memory_format=torch.channels_last) -t2i_pipe.unet = torch.compile(t2i_pipe.unet, mode="reduce-overhead", fullgraph=True) -``` - -After compilation you should see a very fast inference time. For more information, -feel free to have a look at [Our PyTorch 2.0 benchmark](https://huggingface.co/docs/diffusers/main/en/optimization/torch2.0). +The original codebase can be found at [ai-forever/Kandinsky-2](https://github.com/ai-forever/Kandinsky-2). -To generate images directly from a single pipeline, you can use [`KandinskyV22CombinedPipeline`], [`KandinskyV22Img2ImgCombinedPipeline`], [`KandinskyV22InpaintCombinedPipeline`]. -These combined pipelines wrap the [`KandinskyV22PriorPipeline`] and [`KandinskyV22Pipeline`], [`KandinskyV22Img2ImgPipeline`], [`KandinskyV22InpaintPipeline`] respectively into a single -pipeline for a simpler user experience +Check out the [Kandinsky Community](https://huggingface.co/kandinsky-community) organization on the Hub for the official model checkpoints for tasks like text-to-image, image-to-image, and inpainting. -## Available Pipelines: - -| Pipeline | Tasks | -|---|---| -| [pipeline_kandinsky2_2.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py) | *Text-to-Image Generation* | -| [pipeline_kandinsky2_2_combined.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py) | *End-to-end Text-to-Image, image-to-image, Inpainting Generation* | -| [pipeline_kandinsky2_2_inpaint.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpaint.py) | *Image-Guided Image Generation* | -| [pipeline_kandinsky2_2_img2img.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py) | *Image-Guided Image Generation* | -| [pipeline_kandinsky2_2_controlnet.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py) | *Image-Guided Image Generation* | -| [pipeline_kandinsky2_2_controlnet_img2img.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py) | *Image-Guided Image Generation* | +## KandinskyV22PriorPipeline - -### KandinskyV22Pipeline - -[[autodoc]] KandinskyV22Pipeline +[[autodoc]] KandinskyV22PriorPipeline - all - __call__ + - interpolate -### KandinskyV22ControlnetPipeline +## KandinskyV22Pipeline -[[autodoc]] KandinskyV22ControlnetPipeline +[[autodoc]] KandinskyV22Pipeline - all - __call__ -### KandinskyV22ControlnetImg2ImgPipeline +## KandinskyV22CombinedPipeline -[[autodoc]] KandinskyV22ControlnetImg2ImgPipeline +[[autodoc]] KandinskyV22CombinedPipeline - all - __call__ -### KandinskyV22Img2ImgPipeline +## KandinskyV22ControlnetPipeline -[[autodoc]] KandinskyV22Img2ImgPipeline +[[autodoc]] KandinskyV22ControlnetPipeline - all - __call__ -### KandinskyV22InpaintPipeline +## KandinskyV22PriorEmb2EmbPipeline -[[autodoc]] KandinskyV22InpaintPipeline +[[autodoc]] KandinskyV22PriorEmb2EmbPipeline - all - __call__ + - interpolate -### KandinskyV22PriorPipeline +## KandinskyV22Img2ImgPipeline -[[autodoc]] KandinskyV22PriorPipeline +[[autodoc]] KandinskyV22Img2ImgPipeline - all - __call__ - - interpolate -### KandinskyV22PriorEmb2EmbPipeline +## KandinskyV22Img2ImgCombinedPipeline -[[autodoc]] KandinskyV22PriorEmb2EmbPipeline +[[autodoc]] KandinskyV22Img2ImgCombinedPipeline - all - __call__ - - interpolate -### KandinskyV22CombinedPipeline +## KandinskyV22ControlnetImg2ImgPipeline -[[autodoc]] KandinskyV22CombinedPipeline +[[autodoc]] KandinskyV22ControlnetImg2ImgPipeline - all - __call__ -### KandinskyV22Img2ImgCombinedPipeline +## KandinskyV22InpaintPipeline -[[autodoc]] KandinskyV22Img2ImgCombinedPipeline +[[autodoc]] KandinskyV22InpaintPipeline - all - __call__ -### KandinskyV22InpaintCombinedPipeline +## KandinskyV22InpaintCombinedPipeline [[autodoc]] KandinskyV22InpaintCombinedPipeline - all diff --git a/docs/source/en/using-diffusers/kandinsky.md b/docs/source/en/using-diffusers/kandinsky.md new file mode 100644 index 000000000000..4ca544270766 --- /dev/null +++ b/docs/source/en/using-diffusers/kandinsky.md @@ -0,0 +1,692 @@ +# Kandinsky + +[[open-in-colab]] + +The Kandinsky models are a series of multilingual text-to-image generation models. The Kandinsky 2.0 model uses two multilingual text encoders and concatenates those results for the UNet. + +[Kandinsky 2.1](../api/pipelines/kandinsky) changes the architecture to include an image prior model ([`CLIP`](https://huggingface.co/docs/transformers/model_doc/clip)) to generate a mapping between text and image embeddings. The mapping provides better text-image alignment and it is used with the text embeddings during training, leading to higher quality results. Finally, Kandinsky 2.1 uses a [Modulating Quantized Vectors (MoVQ)](https://huggingface.co/papers/2209.09002) decoder - which adds a spatial conditional normalization layer to increase photorealism - to decode the latents into images. + +[Kandinsky 2.2](../api/pipelines/kandinsky_v22) improves on the previous model by replacing the image encoder of the image prior model with a larger CLIP-ViT-G model to improve quality. The image prior model was also retrained on images with different resolutions and aspect ratios to generate higher-resolution images and different image sizes. + +This guide will show you how to use the Kandinsky models for text-to-image, image-to-image, inpainting, interpolation, and more. + +Before you begin, make sure you have the following libraries installed: + +```py +# uncomment to install the necessary libraries in Colab +#!pip install transformers accelerate safetensors +``` + + + +Kandinsky 2.1 and 2.2 usage is very similar! The only difference is Kandinsky 2.2 doesn't accept `prompt` as an input when decoding the latents. Instead, Kandinsky 2.2 only accepts `image_embeds` during decoding. + + + +## Text-to-image + +To use the Kandinsky models for any task, you always start by setting up the prior pipeline to encode the prompt and generate the image embeddings. The prior pipeline also generates `negative_image_embeds` that correspond to the negative prompt `""`. For better results, you can pass an actual `negative_prompt` to the prior pipeline, but this'll increase the effective batch size of the prior pipeline by 2x. + + + + +```py +from diffusers import KandinskyPriorPipeline, KandinskyPipeline +import torch + +prior_pipeline = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16).to("cuda") +pipeline = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16).to("cuda") + +prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" +negative_prompt = "low quality, bad quality" # optional to include a negative prompt, but results are usually better +image_embeds, negative_image_embeds = prior_pipeline(prompt, negative_prompt, guidance_scale=1.0).to_tuple() +``` + +Now pass all the prompts and embeddings to the [`KandinskyPipeline`] to generate an image: + +```py +image = pipeline(prompt, image_embeds=image_embeds, negative_prompt=negative_prompt, negative_image_embeds=negative_image_embeds, height=768, width=768).images[0] +``` + +
+ +
+ + + + +```py +from diffusers import KandinskyV22PriorPipeline, KandinskyV22Pipeline +import torch + +prior_pipeline = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16).to("cuda") +pipeline = KandinskyV22Pipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16).to("cuda") + +prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" +negative_prompt = "low quality, bad quality" # optional to include a negative prompt, but results are usually better +image_embeds, negative_image_embeds = prior_pipeline(prompt, guidance_scale=1.0).to_tuple() +``` + +Pass the `image_embeds` and `negative_image_embeds` to the [`KandinskyV22Pipeline`] to generate an image: + +```py +image = pipeline(image_embeds=image_embeds, negative_image_embeds=negative_image_embeds, height=768, width=768).images[0] +``` + +
+ +
+ + + + +🤗 Diffusers also provides an end-to-end API with the [`KandinskyCombinedPipeline`] and [`KandinskyV22CombinedPipeline`], meaning you don't have to separately load the prior and text-to-image pipeline. The combined pipeline automatically loads both the prior model and the decoder. You can still set different values for the prior pipeline with the `prior_guidance_scale` and `prior_num_inference_steps` parameters if you want. + +Use the [`AutoPipelineForText2Image`] to automatically call the combined pipelines under the hood: + + + + +```py +from diffusers import AutoPipelineForText2Image +import torch + +pipeline = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16).to("cuda") +pipeline.enable_model_cpu_offload() + +prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" +negative_prompt = "low quality, bad quality" + +image = pipeline(prompt=prompt, negative_prompt=negative_prompt, prior_guidance_scale=1.0, guidance_scale = 4.0, height=768, width=768).images[0] +``` + + + + +```py +from diffusers import AutoPipelineForText2Image +import torch + +pipeline = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16).to("cuda") +pipeline.enable_model_cpu_offload() + +prompt = "A alien cheeseburger creature eating itself, claymation, cinematic, moody lighting" +negative_prompt = "low quality, bad quality" + +image = pipeline(prompt=prompt, negative_prompt=negative_prompt, prior_guidance_scale=1.0, guidance_scale = 4.0, height=768, width=768).images[0] +``` + + + + +## Image-to-image + +For image-to-image, pass the initial image and text prompt to condition the image with to the pipeline. Start by loading the prior pipeline: + + + + +```py +import torch +from diffusers import KandinskyImg2ImgPipeline, KandinskyPriorPipeline + +prior_pipeline = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +pipeline = KandinskyImg2ImgPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +``` + + + + +```py +import torch +from diffusers import KandinskyV22Img2ImgPipeline, KandinskyPriorPipeline + +prior_pipeline = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +pipeline = KandinskyV22Img2ImgPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +``` + + + + +Download an image to condition on: + +```py +from PIL import Image +import requests +from io import BytesIO + +# download image +url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" +response = requests.get(url) +original_image = Image.open(BytesIO(response.content)).convert("RGB") +original_image = original_image.resize((768, 512)) +``` + +
+ +
+ +Generate the `image_embeds` and `negative_image_embeds` with the prior pipeline: + +```py +prompt = "A fantasy landscape, Cinematic lighting" +negative_prompt = "low quality, bad quality" + +image_embeds, negative_image_embeds = prior_pipeline(prompt, negative_prompt).to_tuple() +``` + +Now pass the original image, and all the prompts and embeddings to the pipeline to generate an image: + + + + +```py +image = pipeline(prompt, negative_prompt=negative_prompt, image=original_image, image_embeds=image_emebds, negative_image_embeds=negative_image_embeds, height=768, width=768, strength=0.3).images[0] +``` + +
+ +
+ + + + +```py +image = pipeline(image=original_image, image_embeds=image_emebds, negative_image_embeds=negative_image_embeds, height=768, width=768, strength=0.3).images[0] +``` + +
+ +
+ + + + +🤗 Diffusers also provides an end-to-end API with the [`KandinskyImg2ImgCombinedPipeline`] and [`KandinskyV22Img2ImgCombinedPipeline`], meaning you don't have to separately load the prior and image-to-image pipeline. The combined pipeline automatically loads both the prior model and the decoder. You can still set different values for the prior pipeline with the `prior_guidance_scale` and `prior_num_inference_steps` parameters if you want. + +Use the [`AutoPipelineForImage2Image`] to automatically call the combined pipelines under the hood: + + + + +```py +from diffusers import AutoPipelineForImage2Image +import torch +import requests +from io import BytesIO +from PIL import Image +import os + +pipeline = AutoPipelineForImage2Image.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +pipeline.enable_model_cpu_offload() + +prompt = "A fantasy landscape, Cinematic lighting" +negative_prompt = "low quality, bad quality" + +url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + +response = requests.get(url) +original_image = Image.open(BytesIO(response.content)).convert("RGB") +original_image.thumbnail((768, 768)) + +image = pipeline(prompt=prompt, image=original_image, strength=0.3).images[0] +``` + + + + +```py +from diffusers import AutoPipelineForImage2Image +import torch +import requests +from io import BytesIO +from PIL import Image +import os + +pipeline = AutoPipelineForImage2Image.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16).to("cuda") +pipeline.enable_model_cpu_offload() + +prompt = "A fantasy landscape, Cinematic lighting" +negative_prompt = "low quality, bad quality" + +url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + +response = requests.get(url) +original_image = Image.open(BytesIO(response.content)).convert("RGB") +original_image.thumbnail((768, 768)) + +image = pipeline(prompt=prompt, image=original_image, strength=0.3).images[0] +``` + + + + +## Inpainting + + + +⚠️ The Kandinsky models uses ⬜️ **white pixels** to represent the masked area now instead of black pixels. If you are using [`KandinskyInpaintPipeline`] in production, you need to change the mask to use white pixels: + +```py +# For PIL input +import PIL.ImageOps +mask = PIL.ImageOps.invert(mask) + +# For PyTorch and NumPy input +mask = 1 - mask +``` + + + +For inpainting, you'll need the original image, a mask of the area to replace in the original image, and a text prompt of what to inpaint. Load the prior pipeline: + + + + +```py +from diffusers import KandinskyInpaintPipeline, KandinskyPriorPipeline +from diffusers.utils import load_image +import torch +import numpy as np + +prior_pipeline = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +pipeline = KandinskyInpaintPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +``` + + + + +```py +from diffusers import KandinskyV22InpaintPipeline, KandinskyV22PriorPipeline +from diffusers.utils import load_image +import torch +import numpy as np + +prior_pipeline = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +pipeline = KandinskyV22InpaintPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder-inpaint", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +``` + + + + +Load an initial image and create a mask: + +```py +init_image = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/cat.png") +mask = np.zeros((768, 768), dtype=np.float32) +# mask area above cat's head +mask[:250, 250:-250] = 1 +``` + +Generate the embeddings with the prior pipeline: + +```py +prompt = "a hat" +prior_output = prior_pipeline(prompt) +``` + +Now pass the initial image, mask, and prompt and embeddings to the pipeline to generate an image: + + + + +```py +image = pipeline(prompt, image=init_image, mask_image=mask, **prior_output, height=768, width=768, num_inference_steps=150).images[0] +``` + +
+ +
+ + + + +```py +image = pipeline(image=init_image, mask_image=mask, **prior_output, height=768, width=768, num_inference_steps=150).images[0] +``` + +
+ +
+ + + + +You can also use the end-to-end [`KandinskyInpaintCombinedPipeline`] and [`KandinskyV22InpaintCombinedPipeline`] to call the prior and decoder pipelines together under the hood. Use the [`AutoPipelineForInpainting`] for this: + + + + +```py +import torch +from diffusers import AutoPipelineForInpainting + +pipe = AutoPipelineForInpainting.from_pretrained("kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16) +pipe.enable_model_cpu_offload() + +prompt = "a hat" + +image = pipe(prompt=prompt, image=original_image, mask_image=mask).images[0] +``` + + + + +```py +import torch +from diffusers import AutoPipelineForInpainting + +pipe = AutoPipelineForInpainting.from_pretrained("kandinsky-community/kandinsky-2-2-decoder-inpaint", torch_dtype=torch.float16) +pipe.enable_model_cpu_offload() + +prompt = "a hat" + +image = pipe(prompt=prompt, image=original_image, mask_image=mask).images[0] +``` + + + + +## Interpolation + +Interpolation allows you to explore the latent space between the image and text embeddings which is a cool way to see some of the prior model's intermediate outputs. Load the prior pipeline and two images you'd like to interpolate: + + + + +```py +from diffusers import KandinskyPriorPipeline, KandinskyPipeline +from diffusers.utils import load_image +import PIL +import torch + +prior_pipeline = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +img_1 = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/cat.png") +img_2 = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/starry_night.jpeg") +``` + + + + +```py +from diffusers import KandinskyV22PriorPipeline, KandinskyV22Pipeline +from diffusers.utils import load_image +import PIL +import torch + +prior_pipeline = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16, use_safetensors=True).to("cuda") +img_1 = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/cat.png") +img_2 = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/starry_night.jpeg") +``` + + + + +
+
+ +
a cat
+
+
+ +
Van Gogh's Starry Night painting
+
+
+ +Specify the text or images to interpolate, and set the weights for each text or image. Experiment with the weights to see how they affect the interpolation! + +```py +images_texts = ["a cat", img1, img2] +weights = [0.3, 0.3, 0.4] +``` + +Call the `interpolate` function to generate the embeddings, and then pass them to the pipeline to generate the image: + + + + +```py +# prompt can be left empty +prompt = "" +prior_out = prior_pipeline.interpolate(images_texts, weights) + +pipeline = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16, use_safetensors=True).to("cuda") + +image = pipeline(prompt, **prior_out, height=768, width=768).images[0] +image +``` + +
+ +
+ + + + +```py +# prompt can be left empty +prompt = "" +prior_out = prior_pipeline.interpolate(images_texts, weights) + +pipeline = KandinskyV22Pipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16, use_safetensors=True).to("cuda") + +image = pipeline(prompt, **prior_out, height=768, width=768).images[0] +image +``` + +
+ +
+ + + + +## ControlNet + + + +⚠️ ControlNet is only supported for Kandinsky 2.2! + + + +ControlNet enables conditioning large pretrained diffusion models with additional inputs such as a depth map or edge detection. For example, you can condition Kandinsky 2.2 with a depth map so the model understands and preserves the structure of the depth image. + +Let's load an image and extract it's depth map: + +```py +from diffusers.utils import load_image + +img = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinskyv22/cat.png" +).resize((768, 768)) +``` + +
+ +
+ +Then you can use the `depth-estimation` [`~transformers.Pipeline`] from 🤗 Transformers to process the image and retrieve the depth map: + +```py +import torch +import numpy as np + +from transformers import pipeline +from diffusers.utils import load_image + + +def make_hint(image, depth_estimator): + image = depth_estimator(image)["depth"] + image = np.array(image) + image = image[:, :, None] + image = np.concatenate([image, image, image], axis=2) + detected_map = torch.from_numpy(image).float() / 255.0 + hint = detected_map.permute(2, 0, 1) + return hint + + +depth_estimator = pipeline("depth-estimation") +hint = make_hint(img, depth_estimator).unsqueeze(0).half().to("cuda") +``` + +### Text-to-image [[controlnet-text-to-image]] + +Load the prior pipeline and the [`KandinskyV22ControlnetPipeline`]: + +```py +from diffusers import KandinskyV22PriorPipeline, KandinskyV22ControlnetPipeline + +prior_pipeline = KandinskyV22PriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16, use_safetensors=True +)to("cuda") + +pipeline = KandinskyV22ControlnetPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16, use_safetensors=True +).to("cuda") +``` + +Generate the image embeddings from a prompt and negative prompt: + +```py +prompt = "A robot, 4k photo" + +negative_prior_prompt = "lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature" + +generator = torch.Generator(device="cuda").manual_seed(43) +image_emb, zero_image_emb = pipe_prior( + prompt=prompt, negative_prompt=negative_prior_prompt, generator=generator +).to_tuple() +``` + +Finally, pass the image embeddings and the depth image to the [`KandinskyV22ControlnetPipeline`] to generate an image: + +```py +image = pipeline(image_embeds=image_emb, negative_image_embeds=zero_image_emb, hint=hint, num_inference_steps=50, generator=generator, height=768, width=768).images[0] +image +``` + +
+ +
+ +### Image-to-image [[controlnet-image-to-image]] + +For image-to-image with ControlNet, you'll need to use the: + +- [`KandinskyV22PriorEmb2EmbPipeline`] to generate the image embeddings from a text prompt and an image +- [`KandinskyV22ControlnetImg2ImgPipeline`] to generate an image from the initial image and the image embeddings + +Process and extract a depth map of an initial image of a cat with the `depth-estimation` [`~transformers.Pipeline`] from 🤗 Transformers: + +```py +import torch +import numpy as np + +from diffusers import KandinskyV22PriorEmb2EmbPipeline, KandinskyV22ControlnetImg2ImgPipeline +from diffusers.utils import load_image +from transformers import pipeline + +img = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinskyv22/cat.png" +).resize((768, 768)) + + +def make_hint(image, depth_estimator): + image = depth_estimator(image)["depth"] + image = np.array(image) + image = image[:, :, None] + image = np.concatenate([image, image, image], axis=2) + detected_map = torch.from_numpy(image).float() / 255.0 + hint = detected_map.permute(2, 0, 1) + return hint + + +depth_estimator = pipeline("depth-estimation") +hint = make_hint(img, depth_estimator).unsqueeze(0).half().to("cuda") +``` + +Load the prior pipeline and the [`KandinskyV22ControlnetImg2ImgPipeline`]: + +```py +prior_pipeline = KandinskyV22PriorEmb2EmbPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16, use_safetensors=True +).to("cuda") + +pipeline = KandinskyV22ControlnetImg2ImgPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 +).to("cuda") +``` + +Pass a text prompt and the initial image to the prior pipeline to generate the image embeddings: + +```py +prompt = "A robot, 4k photo" +negative_prior_prompt = "lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature" + +generator = torch.Generator(device="cuda").manual_seed(43) + +img_emb = pipe_prior(prompt=prompt, image=img, strength=0.85, generator=generator) +negative_emb = pipe_prior(prompt=negative_prior_prompt, image=img, strength=1, generator=generator) +``` + +Now you can run the [`KandinskyV22ControlnetImg2ImgPipeline`] to generate an image from the initial image and the image embeddings: + +```py +image = pipeline(image=img, strength=0.5, image_embeds=img_emb.image_embeds, negative_image_embeds=negative_emb.image_embeds, hint=hint, num_inference_steps=50, generator=generator, height=768, width=768).images[0] +image +``` + +
+ +
+ +## Optimizations + +Kandinsky is unique because it requires a prior pipeline to generate the mappings, and a second pipeline to decode the latents into an image. Optimization efforts should be focused on the second pipeline because that is where the bulk of the computation is done. Here are some tips to improve Kandinsky during inference. + +1. Enable [xFormers](https://moon-ci-docs.huggingface.co/optimization/xformers) if you're using PyTorch < 2.0: + +```diff + from diffusers import DiffusionPipeline + import torch + + pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) ++ pipe.enable_xformers_memory_efficient_attention() +``` + +2. Enable `torch.compile` if you're using PyTorch 2.0 to automatically use scaled dot-product attention (SDPA): + +```diff + pipe.unet.to(memory_format=torch.channels_last) ++ pipe.unet = torch.compile(pipe.unet, mode="reduced-overhead", fullgraph=True) + + pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) ++ pipe.enable_xformers_memory_efficient_attention() +``` + +This is the same as explicitly setting the attention processor to use [`~models.attention_processor.AttnAddedKVProcessor2_0`]: + +```py +from diffusers.models.attention_processor import AttnAddedKVProcessor2_0 + +pipe.unet.set_attn_processor(AttnAddedKVProcessor2_0()) +``` + +3. Offload the model to the CPU with [`~KandinskyPriorPipeline.enable_model_cpu_offload`] to avoid out-of-memory errors: + +```diff + from diffusers import DiffusionPipeline + import torch + + pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) ++ pipe.enable_model_cpu_offload() +``` + +4. By default, the text-to-image pipeline uses the [`DDIMScheduler`] but you can replace it with another scheduler like [`DDPMScheduler`] to see how that affects the tradeoff between inference speed and image quality: + +```py +from diffusers import DDPMSCheduler + +scheduler = DDPMScheduler.from_pretrained("kandinsky-community/kandinsky-2-1", subfolder="ddpm_scheduler") +pipe = DiffusionPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", scheduler=scheduler, torch_dtype=torch.float16, use_safetensors=True).to("cuda") +``` \ No newline at end of file