|
| 1 | +# Vendored from https://raw.githubusercontent.com/CompVis/taming-transformers/24268930bf1dce879235a7fddd0b2355b84d7ea6/taming/modules/vqvae/quantize.py, |
| 2 | +# where the license is as follows: |
| 3 | +# |
| 4 | +# Copyright (c) 2020 Patrick Esser and Robin Rombach and Björn Ommer |
| 5 | +# |
| 6 | +# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | +# of this software and associated documentation files (the "Software"), to deal |
| 8 | +# in the Software without restriction, including without limitation the rights |
| 9 | +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | +# copies of the Software, and to permit persons to whom the Software is |
| 11 | +# furnished to do so, subject to the following conditions: |
| 12 | +# |
| 13 | +# The above copyright notice and this permission notice shall be included in all |
| 14 | +# copies or substantial portions of the Software. |
| 15 | +# |
| 16 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 17 | +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 18 | +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
| 19 | +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, |
| 20 | +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
| 21 | +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE |
| 22 | +# OR OTHER DEALINGS IN THE SOFTWARE./ |
| 23 | + |
| 24 | +import torch |
| 25 | +import torch.nn as nn |
| 26 | +import numpy as np |
| 27 | +from einops import rearrange |
| 28 | + |
| 29 | + |
| 30 | +class VectorQuantizer2(nn.Module): |
| 31 | + """ |
| 32 | + Improved version over VectorQuantizer, can be used as a drop-in replacement. Mostly |
| 33 | + avoids costly matrix multiplications and allows for post-hoc remapping of indices. |
| 34 | + """ |
| 35 | + |
| 36 | + # NOTE: due to a bug the beta term was applied to the wrong term. for |
| 37 | + # backwards compatibility we use the buggy version by default, but you can |
| 38 | + # specify legacy=False to fix it. |
| 39 | + def __init__(self, n_e, e_dim, beta, remap=None, unknown_index="random", |
| 40 | + sane_index_shape=False, legacy=True): |
| 41 | + super().__init__() |
| 42 | + self.n_e = n_e |
| 43 | + self.e_dim = e_dim |
| 44 | + self.beta = beta |
| 45 | + self.legacy = legacy |
| 46 | + |
| 47 | + self.embedding = nn.Embedding(self.n_e, self.e_dim) |
| 48 | + self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e) |
| 49 | + |
| 50 | + self.remap = remap |
| 51 | + if self.remap is not None: |
| 52 | + self.register_buffer("used", torch.tensor(np.load(self.remap))) |
| 53 | + self.re_embed = self.used.shape[0] |
| 54 | + self.unknown_index = unknown_index # "random" or "extra" or integer |
| 55 | + if self.unknown_index == "extra": |
| 56 | + self.unknown_index = self.re_embed |
| 57 | + self.re_embed = self.re_embed + 1 |
| 58 | + print(f"Remapping {self.n_e} indices to {self.re_embed} indices. " |
| 59 | + f"Using {self.unknown_index} for unknown indices.") |
| 60 | + else: |
| 61 | + self.re_embed = n_e |
| 62 | + |
| 63 | + self.sane_index_shape = sane_index_shape |
| 64 | + |
| 65 | + def remap_to_used(self, inds): |
| 66 | + ishape = inds.shape |
| 67 | + assert len(ishape) > 1 |
| 68 | + inds = inds.reshape(ishape[0], -1) |
| 69 | + used = self.used.to(inds) |
| 70 | + match = (inds[:, :, None] == used[None, None, ...]).long() |
| 71 | + new = match.argmax(-1) |
| 72 | + unknown = match.sum(2) < 1 |
| 73 | + if self.unknown_index == "random": |
| 74 | + new[unknown] = torch.randint(0, self.re_embed, size=new[unknown].shape).to(device=new.device) |
| 75 | + else: |
| 76 | + new[unknown] = self.unknown_index |
| 77 | + return new.reshape(ishape) |
| 78 | + |
| 79 | + def unmap_to_all(self, inds): |
| 80 | + ishape = inds.shape |
| 81 | + assert len(ishape) > 1 |
| 82 | + inds = inds.reshape(ishape[0], -1) |
| 83 | + used = self.used.to(inds) |
| 84 | + if self.re_embed > self.used.shape[0]: # extra token |
| 85 | + inds[inds >= self.used.shape[0]] = 0 # simply set to zero |
| 86 | + back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds) |
| 87 | + return back.reshape(ishape) |
| 88 | + |
| 89 | + def forward(self, z, temp=None, rescale_logits=False, return_logits=False): |
| 90 | + assert temp is None or temp == 1.0, "Only for interface compatible with Gumbel" |
| 91 | + assert rescale_logits is False, "Only for interface compatible with Gumbel" |
| 92 | + assert return_logits is False, "Only for interface compatible with Gumbel" |
| 93 | + # reshape z -> (batch, height, width, channel) and flatten |
| 94 | + z = rearrange(z, 'b c h w -> b h w c').contiguous() |
| 95 | + z_flattened = z.view(-1, self.e_dim) |
| 96 | + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z |
| 97 | + |
| 98 | + d = torch.sum(z_flattened ** 2, dim=1, keepdim=True) + \ |
| 99 | + torch.sum(self.embedding.weight ** 2, dim=1) - 2 * \ |
| 100 | + torch.einsum('bd,dn->bn', z_flattened, rearrange(self.embedding.weight, 'n d -> d n')) |
| 101 | + |
| 102 | + min_encoding_indices = torch.argmin(d, dim=1) |
| 103 | + z_q = self.embedding(min_encoding_indices).view(z.shape) |
| 104 | + perplexity = None |
| 105 | + min_encodings = None |
| 106 | + |
| 107 | + # compute loss for embedding |
| 108 | + if not self.legacy: |
| 109 | + loss = self.beta * torch.mean((z_q.detach() - z) ** 2) + \ |
| 110 | + torch.mean((z_q - z.detach()) ** 2) |
| 111 | + else: |
| 112 | + loss = torch.mean((z_q.detach() - z) ** 2) + self.beta * \ |
| 113 | + torch.mean((z_q - z.detach()) ** 2) |
| 114 | + |
| 115 | + # preserve gradients |
| 116 | + z_q = z + (z_q - z).detach() |
| 117 | + |
| 118 | + # reshape back to match original input shape |
| 119 | + z_q = rearrange(z_q, 'b h w c -> b c h w').contiguous() |
| 120 | + |
| 121 | + if self.remap is not None: |
| 122 | + min_encoding_indices = min_encoding_indices.reshape(z.shape[0], -1) # add batch axis |
| 123 | + min_encoding_indices = self.remap_to_used(min_encoding_indices) |
| 124 | + min_encoding_indices = min_encoding_indices.reshape(-1, 1) # flatten |
| 125 | + |
| 126 | + if self.sane_index_shape: |
| 127 | + min_encoding_indices = min_encoding_indices.reshape( |
| 128 | + z_q.shape[0], z_q.shape[2], z_q.shape[3]) |
| 129 | + |
| 130 | + return z_q, loss, (perplexity, min_encodings, min_encoding_indices) |
| 131 | + |
| 132 | + def get_codebook_entry(self, indices, shape): |
| 133 | + # shape specifying (batch, height, width, channel) |
| 134 | + if self.remap is not None: |
| 135 | + indices = indices.reshape(shape[0], -1) # add batch axis |
| 136 | + indices = self.unmap_to_all(indices) |
| 137 | + indices = indices.reshape(-1) # flatten again |
| 138 | + |
| 139 | + # get quantized latent vectors |
| 140 | + z_q = self.embedding(indices) |
| 141 | + |
| 142 | + if shape is not None: |
| 143 | + z_q = z_q.view(shape) |
| 144 | + # reshape back to match original input shape |
| 145 | + z_q = z_q.permute(0, 3, 1, 2).contiguous() |
| 146 | + |
| 147 | + return z_q |
0 commit comments