Skip to content

Commit b02d0d6

Browse files
patrickvonplatenpatrickvonplaten
committed
merge
2 parents 49257b4 + 02cdd68 commit b02d0d6

25 files changed

Lines changed: 3630 additions & 122 deletions

‎models/vision/ddim/README.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<!--Copyright 2022 The HuggingFace Team. All rights reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
4+
the License. You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
9+
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
10+
specific language governing permissions and limitations under the License.
11+
-->
12+
13+
# Denoising Diffusion Implicit Models (DDIM)
14+
15+
## Overview
16+
17+
DDPM was proposed in [Denoising Diffusion Implicit Models](https://arxiv.org/abs/2010.02502) by *Jiaming Song, Chenlin Meng, Stefano Ermon*
18+
19+
The abstract from the paper is the following:
20+
21+
*Denoising diffusion probabilistic models (DDPMs) have achieved high quality image generation without adversarial training, yet they require simulating a Markov chain for many steps to produce a sample. To accelerate sampling, we present denoising diffusion implicit models (DDIMs), a more efficient class of iterative implicit probabilistic models with the same training procedure as DDPMs. In DDPMs, the generative process is defined as the reverse of a Markovian diffusion process. We construct a class of non-Markovian diffusion processes that lead to the same training objective, but whose reverse process can be much faster to sample from. We empirically demonstrate that DDIMs can produce high quality samples 10× to 50× faster in terms of wall-clock time compared to DDPMs, allow us to trade off computation for sample quality, and can perform semantically meaningful image interpolation directly in the latent space.*
22+
23+
Tips:
24+
25+
- ...
26+
- ...
27+
28+
This model was contributed by [???](https://huggingface.co/???). The original code can be found [here](https://github.com/hojonathanho/diffusion).

‎models/vision/ddim/example.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env python3
2+
import os
3+
import pathlib
4+
from modeling_ddim import DDIM
5+
import PIL.Image
6+
import numpy as np
7+
8+
model_ids = ["ddim-celeba-hq", "ddim-lsun-church", "ddim-lsun-bedroom"]
9+
10+
for model_id in model_ids:
11+
path = os.path.join("/home/patrick/images/hf", model_id)
12+
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
13+
14+
ddpm = DDIM.from_pretrained("fusing/" + model_id)
15+
image = ddpm(batch_size=4)
16+
17+
image_processed = image.cpu().permute(0, 2, 3, 1)
18+
image_processed = (image_processed + 1.0) * 127.5
19+
image_processed = image_processed.numpy().astype(np.uint8)
20+
21+
for i in range(image_processed.shape[0]):
22+
image_pil = PIL.Image.fromarray(image_processed[i])
23+
image_pil.save(os.path.join(path, f"image_{i}.png"))

‎models/vision/ddim/modeling_ddim.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# Copyright 2022 The HuggingFace Team. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
14+
# limitations under the License.
15+
16+
17+
from diffusers import DiffusionPipeline
18+
import tqdm
19+
import torch
20+
21+
22+
class DDIM(DiffusionPipeline):
23+
24+
def __init__(self, unet, noise_scheduler):
25+
super().__init__()
26+
self.register_modules(unet=unet, noise_scheduler=noise_scheduler)
27+
28+
def __call__(self, batch_size=1, generator=None, torch_device=None, eta=0.0, num_inference_steps=50):
29+
# eta corresponds to η in paper and should be between [0, 1]
30+
if torch_device is None:
31+
torch_device = "cuda" if torch.cuda.is_available() else "cpu"
32+
33+
num_trained_timesteps = self.noise_scheduler.num_timesteps
34+
inference_step_times = range(0, num_trained_timesteps, num_trained_timesteps // num_inference_steps)
35+
36+
self.unet.to(torch_device)
37+
image = self.noise_scheduler.sample_noise((batch_size, self.unet.in_channels, self.unet.resolution, self.unet.resolution), device=torch_device, generator=generator)
38+
39+
for t in tqdm.tqdm(reversed(range(num_inference_steps)), total=num_inference_steps):
40+
# get actual t and t-1
41+
train_step = inference_step_times[t]
42+
prev_train_step = inference_step_times[t - 1] if t > 0 else - 1
43+
44+
# compute alphas
45+
alpha_prod_t = self.noise_scheduler.get_alpha_prod(train_step)
46+
alpha_prod_t_prev = self.noise_scheduler.get_alpha_prod(prev_train_step)
47+
alpha_prod_t_rsqrt = 1 / alpha_prod_t.sqrt()
48+
alpha_prod_t_prev_rsqrt = 1 / alpha_prod_t_prev.sqrt()
49+
beta_prod_t_sqrt = (1 - alpha_prod_t).sqrt()
50+
beta_prod_t_prev_sqrt = (1 - alpha_prod_t_prev).sqrt()
51+
52+
# compute relevant coefficients
53+
coeff_1 = (alpha_prod_t_prev - alpha_prod_t).sqrt() * alpha_prod_t_prev_rsqrt * beta_prod_t_prev_sqrt / beta_prod_t_sqrt * eta
54+
coeff_2 = ((1 - alpha_prod_t_prev) - coeff_1 ** 2).sqrt()
55+
56+
# model forward
57+
with torch.no_grad():
58+
noise_residual = self.unet(image, train_step)
59+
60+
# predict mean of prev image
61+
pred_mean = alpha_prod_t_rsqrt * (image - beta_prod_t_sqrt * noise_residual)
62+
pred_mean = torch.clamp(pred_mean, -1, 1)
63+
pred_mean = (1 / alpha_prod_t_prev_rsqrt) * pred_mean + coeff_2 * noise_residual
64+
65+
# if eta > 0.0 add noise. Note eta = 1.0 essentially corresponds to DDPM
66+
if eta > 0.0:
67+
noise = self.noise_scheduler.sample_noise(image.shape, device=image.device, generator=generator)
68+
image = pred_mean + coeff_1 * noise
69+
else:
70+
image = pred_mean
71+
72+
return image

‎models/vision/ddim/run_ddpm.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env python3
2+
import torch
3+
4+
from diffusers import GaussianDDPMScheduler, UNetModel
5+
6+
7+
model = UNetModel(dim=64, dim_mults=(1, 2, 4, 8))
8+
9+
diffusion = GaussianDDPMScheduler(model, image_size=128, timesteps=1000, loss_type="l1") # number of steps # L1 or L2
10+
11+
training_images = torch.randn(8, 3, 128, 128) # your images need to be normalized from a range of -1 to +1
12+
loss = diffusion(training_images)
13+
loss.backward()
14+
# after a lot of training
15+
16+
sampled_images = diffusion.sample(batch_size=4)
17+
sampled_images.shape # (4, 3, 128, 128)

‎models/vision/ddim/run_inference.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/env python3
2+
# !pip install diffusers
3+
from modeling_ddim import DDIM
4+
import PIL.Image
5+
import numpy as np
6+
7+
model_id = "fusing/ddpm-cifar10"
8+
model_id = "fusing/ddpm-lsun-bedroom"
9+
10+
# load model and scheduler
11+
ddpm = DDIM.from_pretrained(model_id)
12+
13+
# run pipeline in inference (sample random noise and denoise)
14+
image = ddpm()
15+
16+
# process image to PIL
17+
image_processed = image.cpu().permute(0, 2, 3, 1)
18+
image_processed = (image_processed + 1.0) * 127.5
19+
image_processed = image_processed.numpy().astype(np.uint8)
20+
image_pil = PIL.Image.fromarray(image_processed[0])
21+
22+
# save image
23+
image_pil.save("/home/patrick/images/show.png")

‎models/vision/ddpm/modeling_ddpm.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@
2121

2222
class DDPM(DiffusionPipeline):
2323

24-
modeling_file = "modeling_ddpm.py"
25-
2624
def __init__(self, unet, noise_scheduler):
2725
super().__init__()
2826
self.register_modules(unet=unet, noise_scheduler=noise_scheduler)

‎models/vision/glide/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# References
2+
3+
[GLIDE: Towards Photorealistic Image Generation and Editing with Text-Guided Diffusion Models](https://arxiv.org/pdf/2112.10741.pdf)
4+
[Diffusion Models Beat GANs on Image Synthesis](https://arxiv.org/pdf/2105.05233.pdf)

‎models/vision/glide/convert_weights.py‎

Lines changed: 65 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,28 @@
1-
import argparse
2-
31
import torch
42
from torch import nn
53

6-
from transformers import CLIPTextConfig, CLIPTextModel, GPT2Tokenizer
4+
from diffusers import ClassifierFreeGuidanceScheduler, GlideDDIMScheduler, CLIPTextModel, GLIDETextToImageUNetModel, GLIDESuperResUNetModel
5+
from modeling_glide import GLIDE
6+
from transformers import CLIPTextConfig, GPT2Tokenizer
7+
78

89
# wget https://openaipublic.blob.core.windows.net/diffusion/dec-2021/base.pt
910
state_dict = torch.load("base.pt", map_location="cpu")
1011
state_dict = {k: nn.Parameter(v) for k, v in state_dict.items()}
12+
13+
### Convert the text encoder
14+
1115
config = CLIPTextConfig(
16+
vocab_size=50257,
17+
max_position_embeddings=128,
1218
hidden_size=512,
1319
intermediate_size=2048,
1420
num_hidden_layers=16,
1521
num_attention_heads=8,
16-
max_position_embeddings=128
22+
use_padding_embeddings=True,
1723
)
1824
model = CLIPTextModel(config).eval()
19-
tokenizer = GPT2Tokenizer("./glide-base/vocab.json", "./glide-base/merges.txt", pad_token="<|endoftext|>")
20-
tokenizer.save_pretrained("./glide-base")
25+
tokenizer = GPT2Tokenizer("./glide-base/tokenizer/vocab.json", "./glide-base/tokenizer/merges.txt", pad_token="<|endoftext|>")
2126

2227
hf_encoder = model.text_model
2328

@@ -30,15 +35,8 @@
3035

3136
for layer_idx in range(config.num_hidden_layers):
3237
hf_layer = hf_encoder.encoder.layers[layer_idx]
33-
q_proj, k_proj, v_proj = state_dict[f"transformer.resblocks.{layer_idx}.attn.c_qkv.weight"].chunk(3, dim=0)
34-
q_proj_bias, k_proj_bias, v_proj_bias = state_dict[f"transformer.resblocks.{layer_idx}.attn.c_qkv.bias"].chunk(3, dim=0)
35-
36-
hf_layer.self_attn.q_proj.weight.data = q_proj
37-
hf_layer.self_attn.q_proj.bias.data = q_proj_bias
38-
hf_layer.self_attn.k_proj.weight.data = k_proj
39-
hf_layer.self_attn.k_proj.bias.data = k_proj_bias
40-
hf_layer.self_attn.v_proj.weight.data = v_proj
41-
hf_layer.self_attn.v_proj.bias.data = v_proj_bias
38+
hf_layer.self_attn.qkv_proj.weight = state_dict[f"transformer.resblocks.{layer_idx}.attn.c_qkv.weight"]
39+
hf_layer.self_attn.qkv_proj.bias = state_dict[f"transformer.resblocks.{layer_idx}.attn.c_qkv.bias"]
4240

4341
hf_layer.self_attn.out_proj.weight = state_dict[f"transformer.resblocks.{layer_idx}.attn.c_proj.weight"]
4442
hf_layer.self_attn.out_proj.bias = state_dict[f"transformer.resblocks.{layer_idx}.attn.c_proj.bias"]
@@ -53,8 +51,56 @@
5351
hf_layer.mlp.fc2.weight = state_dict[f"transformer.resblocks.{layer_idx}.mlp.c_proj.weight"]
5452
hf_layer.mlp.fc2.bias = state_dict[f"transformer.resblocks.{layer_idx}.mlp.c_proj.bias"]
5553

56-
inputs = tokenizer(["an oil painting of a corgi", ""], padding="max_length", max_length=128, return_tensors="pt")
57-
with torch.no_grad():
58-
outputs = model(**inputs)
54+
### Convert the Text-to-Image UNet
55+
56+
text2im_model = GLIDETextToImageUNetModel(
57+
in_channels=3,
58+
model_channels=192,
59+
out_channels=6,
60+
num_res_blocks=3,
61+
attention_resolutions=(2, 4, 8),
62+
dropout=0.1,
63+
channel_mult=(1, 2, 3, 4),
64+
num_heads=1,
65+
num_head_channels=64,
66+
num_heads_upsample=1,
67+
use_scale_shift_norm=True,
68+
resblock_updown=True,
69+
transformer_dim=512,
70+
)
71+
72+
text2im_model.load_state_dict(state_dict, strict=False)
73+
74+
text_scheduler = ClassifierFreeGuidanceScheduler(timesteps=1000, beta_schedule="squaredcos_cap_v2")
75+
76+
### Convert the Super-Resolution UNet
77+
78+
# wget https://openaipublic.blob.core.windows.net/diffusion/dec-2021/upsample.pt
79+
ups_state_dict = torch.load("upsample.pt", map_location="cpu")
80+
81+
superres_model = GLIDESuperResUNetModel(
82+
in_channels=6,
83+
model_channels=192,
84+
out_channels=6,
85+
num_res_blocks=2,
86+
attention_resolutions=(8, 16, 32),
87+
dropout=0.1,
88+
channel_mult=(1, 1, 2, 2, 4, 4),
89+
num_heads=1,
90+
num_head_channels=64,
91+
num_heads_upsample=1,
92+
use_scale_shift_norm=True,
93+
resblock_updown=True,
94+
)
95+
96+
superres_model.load_state_dict(ups_state_dict, strict=False)
97+
98+
upscale_scheduler = GlideDDIMScheduler(timesteps=1000, beta_schedule="linear")
99+
100+
glide = GLIDE(text_unet=text2im_model, text_noise_scheduler=text_scheduler, text_encoder=model, tokenizer=tokenizer,
101+
upscale_unet=superres_model, upscale_noise_scheduler=upscale_scheduler)
102+
103+
glide.save_pretrained("./glide-base")
104+
105+
59106

60-
model.save_pretrained("./glide-base")

0 commit comments

Comments
 (0)