import math import numpy as np import torch from typing import List from torch import nn import torch from torch import nn from torch.nn.utils.parametrizations import weight_norm from einops import rearrange # conv modules def WNConv1d(*args, **kwargs): return weight_norm(nn.Conv1d(*args, **kwargs)) def WNConvTranspose1d(*args, **kwargs): return weight_norm(nn.ConvTranspose1d(*args, **kwargs)) @torch.jit.script def snake(x, alpha): shape = x.shape x = x.reshape(shape[0], shape[1], -1) x = x + (alpha + 1e-9).reciprocal() * torch.sin(alpha * x).pow(2) x = x.reshape(shape) return x class Snake1d(nn.Module): def __init__(self, channels): super().__init__() self.alpha = nn.Parameter(torch.ones(1, channels, 1)) def forward(self, x): return snake(x, self.alpha) class ConvNeXtUnit(nn.Module): def __init__(self, dim: int = 16, dilation: int = 1): super().__init__() self.dwconv = nn.Conv1d( dim, dim, kernel_size=7, padding=3 * dilation, groups=dim, dilation=dilation ) self.norm = nn.LayerNorm(dim) self.pwconv1 = nn.Linear(dim, 4 * dim) self.act = nn.GELU() self.pwconv2 = nn.Linear(4 * dim, dim) self.gamma = nn.Parameter(torch.ones(dim, 1) * 1e-6) def forward(self, x): input = x x = self.dwconv(x) x = x.permute(0, 2, 1) # (N, C, L) -> (N, L, C) x = self.norm(x) x = self.pwconv1(x) x = self.act(x) x = self.pwconv2(x) x = x.permute(0, 2, 1) # (N, L, C) -> (N, C, L) # Padding adjustment pad = (input.shape[-1] - x.shape[-1]) // 2 if pad > 0: input = input[..., pad:-pad] elif pad < 0: x = x[..., -pad:pad] x = input + self.gamma * x return x def vae_sample(mean, scale): stdev = nn.functional.softplus(scale) + 1e-4 var = stdev * stdev logvar = torch.log(var) latents = torch.randn_like(mean) * stdev + mean kl = (mean * mean + var - logvar - 1).sum(1).mean() return latents, kl # VAE module class VAEBottleneck(nn.Module): def __init__(self, latent_dim, dim, is_discrete=False, **kwargs): super().__init__() self.project_in = nn.Linear(latent_dim, dim * 2) self.project_out = nn.Linear(dim, latent_dim) if latent_dim != dim else nn.Identity() self.is_discrete = is_discrete def forward(self, x, transpose=True, **kwargs): if transpose: x = rearrange(x, "b d n -> b n d") x = self.project_in(x) mean, scale = x.chunk(2, dim=-1) z, kl = vae_sample(mean, scale) x = self.project_out(z) if transpose: x = rearrange(x, "b n d -> b d n") z = rearrange(z, "b n d -> b d n") return { "z": z, "x": x, "kl": kl, "mean": mean, "scale": scale, } def encode(self, x, return_info=False, **kwargs): if return_info: out = self.forward(x) latents = out["z"] # latents = rearrange(latents, "b n d -> b d n") bottleneck_info = {"kl": float(out["kl"].item())} return latents, bottleneck_info return self.forward(x) def decode(self, x, **kwargs): return x class EncoderBlock(nn.Module): def __init__(self, dim: int = 16, stride: int = 1): super().__init__() self.block = nn.Sequential( ConvNeXtUnit(dim // 2, dilation=1), ConvNeXtUnit(dim // 2, dilation=3), ConvNeXtUnit(dim // 2, dilation=9), Snake1d(dim // 2), WNConv1d( dim // 2, dim, kernel_size=2 * stride, stride=stride, padding=math.ceil(stride / 2), ), ) def forward(self, x): return self.block(x) class Encoder(nn.Module): def __init__( self, d_model: int = 64, strides: list = [2, 4, 8, 8], d_latent: int = 64, ): super().__init__() # Create first convolution self.block = [WNConv1d(2, d_model, kernel_size=7, padding=3)] # Create EncoderBlocks that double channels as they downsample by `stride` for stride in strides: d_model *= 2 self.block += [EncoderBlock(d_model, stride=stride)] # Create last convolution and groupnorm self.block += [ Snake1d(d_model), WNConv1d(d_model, d_latent, kernel_size=3, padding=1), ] # Wrap black into nn.Sequential self.block = nn.Sequential(*self.block) self.enc_dim = d_model def forward(self, x): return self.block(x) class DecoderBlock(nn.Module): def __init__(self, input_dim: int = 16, output_dim: int = 8, stride: int = 1): super().__init__() self.block = nn.Sequential( Snake1d(input_dim), WNConvTranspose1d( input_dim, output_dim, kernel_size=2 * stride, stride=stride, padding=math.floor(stride / 2), ), ConvNeXtUnit(output_dim, dilation=1), ConvNeXtUnit(output_dim, dilation=3), ConvNeXtUnit(output_dim, dilation=9), ) def forward(self, x): return self.block(x) class Decoder(nn.Module): def __init__( self, input_channel, channels, rates, d_out: int = 2, ): super().__init__() # Add first conv layer layers = [WNConv1d(input_channel, channels, kernel_size=7, padding=3)] # Add upsampling + MRF blocks for i, stride in enumerate(rates): input_dim = channels // 2**i output_dim = channels // 2 ** (i + 1) layers += [DecoderBlock(input_dim, output_dim, stride)] # Add final conv layer layers += [ Snake1d(output_dim), WNConv1d(output_dim, d_out, kernel_size=7, padding=3), nn.Tanh(), ] self.model = nn.Sequential(*layers) def forward(self, x): return self.model(x) # convnext codec class ConvNextVAE(nn.Module): def __init__( self, encoder_dim: int = 128, encoder_rates: List[int] = [2, 3, 5, 8, 8], latent_dim: int = 128, decoder_dim: int = 1536, decoder_rates: List[int] = [8, 8, 5, 3, 2], vae_dim: int = 128, sample_rate: int = 48000, **kwargs, ): super().__init__() self.encoder_dim = encoder_dim self.encoder_rates = encoder_rates self.latent_dim = latent_dim self.decoder_dim = decoder_dim self.decoder_rates = decoder_rates self.vae_dim = vae_dim self.sample_rate = sample_rate self.hop_length = np.prod(self.encoder_rates) self.encoder = Encoder(self.encoder_dim, self.encoder_rates, self.latent_dim) # bottleneck self.quantizer = VAEBottleneck(latent_dim=self.latent_dim, dim=self.vae_dim) self.decoder = Decoder( self.vae_dim, self.decoder_dim, self.decoder_rates, ) # S = torch.load("/home/minz/logs/sac_vae_encodecP_peaq_5e5_stereo/last.ckpt") # relevant_weights = {k[6:]: v for k, v in S["state_dict"].items() if k.split(".")[1] in ["encoder", "decoder", "quantizer"]} # self.load_state_dict(relevant_weights, strict=True) # Freeze encoder and quantizer parameters # for param in self.encoder.parameters(): # param.requires_grad = False # for param in self.quantizer.parameters(): # param.requires_grad = False def get_model_hyperparameters(self): return { "kwargs": { "encoder_dim": self.encoder_dim, "encoder_rates": self.encoder_rates, "latent_dim": self.latent_dim, "decoder_dim": self.decoder_dim, "decoder_rates": self.decoder_rates, "vae_dim": self.vae_dim, "sample_rate": self.sample_rate, } } def preprocess(self, audio_data, sample_rate): if sample_rate is None: sample_rate = self.sample_rate assert sample_rate == self.sample_rate length = audio_data.shape[-1] right_pad = math.ceil(length / self.hop_length) * self.hop_length - length audio_data = nn.functional.pad(audio_data, (0, right_pad)) return audio_data def encode( self, audio_data: torch.Tensor, ): """Encode given audio data and return quantized latent codes Parameters ---------- audio_data : Tensor[B x 1 x T] Audio data to encode n_quantizers : int, optional Number of quantizers to use, by default None If None, all quantizers are used. Returns ------- dict A dictionary with the following keys: "z" : Tensor[B x D x T] Quantized continuous representation of input "codes" : Tensor[B x N x T] Codebook indices for each codebook (quantized discrete representation of input) "latents" : Tensor[B x N*D x T] Projected latents (continuous representation of input before quantization) "vq/commitment_loss" : Tensor[1] Commitment loss to train encoder to predict vectors closer to codebook entries "vq/codebook_loss" : Tensor[1] Codebook loss to update the codebook "length" : int Number of samples in input audio """ z = self.encoder(audio_data) return self.quantizer(z) def decode(self, z: torch.Tensor): """Decode given latent codes and return audio data Parameters ---------- z : Tensor[B x D x T] Quantized continuous representation of input length : int, optional Number of samples in output audio, by default None Returns ------- dict A dictionary with the following keys: "audio" : Tensor[B x 1 x length] Decoded audio data. """ return self.decoder(z) def forward( self, audio_data: torch.Tensor, sample_rate: int = None, ): """Model forward pass Parameters ---------- audio_data : Tensor[B x 2 x T] Audio data to encode sample_rate : int, optional Sample rate of audio data in Hz, by default None If None, defaults to `self.sample_rate` Returns ------- dict A dictionary with the following keys: "z" : Tensor[B x D x T] Quantized continuous representation of input "audio" : Tensor[B x 1 x length] Decoded audio data. """ length = audio_data.shape[-1] audio_data = self.preprocess(audio_data, sample_rate) q_res = self.encode(audio_data) x = self.decode(q_res["z"]) return { "audio": x[..., :length], **q_res, } def get_num_params(self): return sum(p.numel() for p in self.parameters() if p.requires_grad)