mirror of
https://github.com/babysor/MockingBird.git
synced 2024-03-22 13:11:31 +08:00
b617a87ee4
* Init ppg extractor and ppg2mel * add preprocess and training * FIx known issues * Update __init__.py Allow to gen audio * Fix length issue * Fix bug of preparing fid * Fix sample issues * Add UI usage of PPG-vc
215 lines
8.1 KiB
Python
215 lines
8.1 KiB
Python
import random
|
|
import numpy as np
|
|
import torch
|
|
from utils.f0_utils import get_cont_lf0
|
|
import resampy
|
|
from .audio_utils import MAX_WAV_VALUE, load_wav, mel_spectrogram
|
|
from librosa.util import normalize
|
|
import os
|
|
|
|
|
|
SAMPLE_RATE=16000
|
|
|
|
def read_fids(fid_list_f):
|
|
with open(fid_list_f, 'r') as f:
|
|
fids = [l.strip().split()[0] for l in f if l.strip()]
|
|
return fids
|
|
|
|
class OneshotVcDataset(torch.utils.data.Dataset):
|
|
def __init__(
|
|
self,
|
|
meta_file: str,
|
|
vctk_ppg_dir: str,
|
|
libri_ppg_dir: str,
|
|
vctk_f0_dir: str,
|
|
libri_f0_dir: str,
|
|
vctk_wav_dir: str,
|
|
libri_wav_dir: str,
|
|
vctk_spk_dvec_dir: str,
|
|
libri_spk_dvec_dir: str,
|
|
min_max_norm_mel: bool = False,
|
|
mel_min: float = None,
|
|
mel_max: float = None,
|
|
ppg_file_ext: str = "ling_feat.npy",
|
|
f0_file_ext: str = "f0.npy",
|
|
wav_file_ext: str = "wav",
|
|
):
|
|
self.fid_list = read_fids(meta_file)
|
|
self.vctk_ppg_dir = vctk_ppg_dir
|
|
self.libri_ppg_dir = libri_ppg_dir
|
|
self.vctk_f0_dir = vctk_f0_dir
|
|
self.libri_f0_dir = libri_f0_dir
|
|
self.vctk_wav_dir = vctk_wav_dir
|
|
self.libri_wav_dir = libri_wav_dir
|
|
self.vctk_spk_dvec_dir = vctk_spk_dvec_dir
|
|
self.libri_spk_dvec_dir = libri_spk_dvec_dir
|
|
|
|
self.ppg_file_ext = ppg_file_ext
|
|
self.f0_file_ext = f0_file_ext
|
|
self.wav_file_ext = wav_file_ext
|
|
|
|
self.min_max_norm_mel = min_max_norm_mel
|
|
if min_max_norm_mel:
|
|
print("[INFO] Min-Max normalize Melspec.")
|
|
assert mel_min is not None
|
|
assert mel_max is not None
|
|
self.mel_max = mel_max
|
|
self.mel_min = mel_min
|
|
|
|
random.seed(1234)
|
|
random.shuffle(self.fid_list)
|
|
print(f'[INFO] Got {len(self.fid_list)} samples.')
|
|
|
|
def __len__(self):
|
|
return len(self.fid_list)
|
|
|
|
def get_spk_dvec(self, fid):
|
|
spk_name = fid
|
|
if spk_name.startswith("p"):
|
|
spk_dvec_path = f"{self.vctk_spk_dvec_dir}{os.sep}{spk_name}.npy"
|
|
else:
|
|
spk_dvec_path = f"{self.libri_spk_dvec_dir}{os.sep}{spk_name}.npy"
|
|
return torch.from_numpy(np.load(spk_dvec_path))
|
|
|
|
def compute_mel(self, wav_path):
|
|
audio, sr = load_wav(wav_path)
|
|
if sr != SAMPLE_RATE:
|
|
audio = resampy.resample(audio, sr, SAMPLE_RATE)
|
|
audio = audio / MAX_WAV_VALUE
|
|
audio = normalize(audio) * 0.95
|
|
audio = torch.FloatTensor(audio).unsqueeze(0)
|
|
melspec = mel_spectrogram(
|
|
audio,
|
|
n_fft=1024,
|
|
num_mels=80,
|
|
sampling_rate=SAMPLE_RATE,
|
|
hop_size=160,
|
|
win_size=1024,
|
|
fmin=80,
|
|
fmax=8000,
|
|
)
|
|
return melspec.squeeze(0).numpy().T
|
|
|
|
def bin_level_min_max_norm(self, melspec):
|
|
# frequency bin level min-max normalization to [-4, 4]
|
|
mel = (melspec - self.mel_min) / (self.mel_max - self.mel_min) * 8.0 - 4.0
|
|
return np.clip(mel, -4., 4.)
|
|
|
|
def __getitem__(self, index):
|
|
fid = self.fid_list[index]
|
|
|
|
# 1. Load features
|
|
if fid.startswith("p"):
|
|
# vctk
|
|
sub = fid.split("_")[0]
|
|
ppg = np.load(f"{self.vctk_ppg_dir}{os.sep}{fid}.{self.ppg_file_ext}")
|
|
f0 = np.load(f"{self.vctk_f0_dir}{os.sep}{fid}.{self.f0_file_ext}")
|
|
mel = self.compute_mel(f"{self.vctk_wav_dir}{os.sep}{sub}{os.sep}{fid}.{self.wav_file_ext}")
|
|
else:
|
|
# aidatatang
|
|
sub = fid[5:10]
|
|
ppg = np.load(f"{self.libri_ppg_dir}{os.sep}{fid}.{self.ppg_file_ext}")
|
|
f0 = np.load(f"{self.libri_f0_dir}{os.sep}{fid}.{self.f0_file_ext}")
|
|
mel = self.compute_mel(f"{self.libri_wav_dir}{os.sep}{sub}{os.sep}{fid}.{self.wav_file_ext}")
|
|
if self.min_max_norm_mel:
|
|
mel = self.bin_level_min_max_norm(mel)
|
|
|
|
f0, ppg, mel = self._adjust_lengths(f0, ppg, mel, fid)
|
|
spk_dvec = self.get_spk_dvec(fid)
|
|
|
|
# 2. Convert f0 to continuous log-f0 and u/v flags
|
|
uv, cont_lf0 = get_cont_lf0(f0, 10.0, False)
|
|
# cont_lf0 = (cont_lf0 - np.amin(cont_lf0)) / (np.amax(cont_lf0) - np.amin(cont_lf0))
|
|
# cont_lf0 = self.utt_mvn(cont_lf0)
|
|
lf0_uv = np.concatenate([cont_lf0[:, np.newaxis], uv[:, np.newaxis]], axis=1)
|
|
|
|
# uv, cont_f0 = convert_continuous_f0(f0)
|
|
# cont_f0 = (cont_f0 - np.amin(cont_f0)) / (np.amax(cont_f0) - np.amin(cont_f0))
|
|
# lf0_uv = np.concatenate([cont_f0[:, np.newaxis], uv[:, np.newaxis]], axis=1)
|
|
|
|
# 3. Convert numpy array to torch.tensor
|
|
ppg = torch.from_numpy(ppg)
|
|
lf0_uv = torch.from_numpy(lf0_uv)
|
|
mel = torch.from_numpy(mel)
|
|
|
|
return (ppg, lf0_uv, mel, spk_dvec, fid)
|
|
|
|
def check_lengths(self, f0, ppg, mel, fid):
|
|
LEN_THRESH = 10
|
|
assert abs(len(ppg) - len(f0)) <= LEN_THRESH, \
|
|
f"{abs(len(ppg) - len(f0))}: for file {fid}"
|
|
assert abs(len(mel) - len(f0)) <= LEN_THRESH, \
|
|
f"{abs(len(mel) - len(f0))}: for file {fid}"
|
|
|
|
def _adjust_lengths(self, f0, ppg, mel, fid):
|
|
self.check_lengths(f0, ppg, mel, fid)
|
|
min_len = min(
|
|
len(f0),
|
|
len(ppg),
|
|
len(mel),
|
|
)
|
|
f0 = f0[:min_len]
|
|
ppg = ppg[:min_len]
|
|
mel = mel[:min_len]
|
|
return f0, ppg, mel
|
|
|
|
class MultiSpkVcCollate():
|
|
"""Zero-pads model inputs and targets based on number of frames per step
|
|
"""
|
|
def __init__(self, n_frames_per_step=1, give_uttids=False,
|
|
f02ppg_length_ratio=1, use_spk_dvec=False):
|
|
self.n_frames_per_step = n_frames_per_step
|
|
self.give_uttids = give_uttids
|
|
self.f02ppg_length_ratio = f02ppg_length_ratio
|
|
self.use_spk_dvec = use_spk_dvec
|
|
|
|
def __call__(self, batch):
|
|
batch_size = len(batch)
|
|
# Prepare different features
|
|
ppgs = [x[0] for x in batch]
|
|
lf0_uvs = [x[1] for x in batch]
|
|
mels = [x[2] for x in batch]
|
|
fids = [x[-1] for x in batch]
|
|
if len(batch[0]) == 5:
|
|
spk_ids = [x[3] for x in batch]
|
|
if self.use_spk_dvec:
|
|
# use d-vector
|
|
spk_ids = torch.stack(spk_ids).float()
|
|
else:
|
|
# use one-hot ids
|
|
spk_ids = torch.LongTensor(spk_ids)
|
|
# Pad features into chunk
|
|
ppg_lengths = [x.shape[0] for x in ppgs]
|
|
mel_lengths = [x.shape[0] for x in mels]
|
|
max_ppg_len = max(ppg_lengths)
|
|
max_mel_len = max(mel_lengths)
|
|
if max_mel_len % self.n_frames_per_step != 0:
|
|
max_mel_len += (self.n_frames_per_step - max_mel_len % self.n_frames_per_step)
|
|
ppg_dim = ppgs[0].shape[1]
|
|
mel_dim = mels[0].shape[1]
|
|
ppgs_padded = torch.FloatTensor(batch_size, max_ppg_len, ppg_dim).zero_()
|
|
mels_padded = torch.FloatTensor(batch_size, max_mel_len, mel_dim).zero_()
|
|
lf0_uvs_padded = torch.FloatTensor(batch_size, self.f02ppg_length_ratio * max_ppg_len, 2).zero_()
|
|
stop_tokens = torch.FloatTensor(batch_size, max_mel_len).zero_()
|
|
for i in range(batch_size):
|
|
cur_ppg_len = ppgs[i].shape[0]
|
|
cur_mel_len = mels[i].shape[0]
|
|
ppgs_padded[i, :cur_ppg_len, :] = ppgs[i]
|
|
lf0_uvs_padded[i, :self.f02ppg_length_ratio*cur_ppg_len, :] = lf0_uvs[i]
|
|
mels_padded[i, :cur_mel_len, :] = mels[i]
|
|
stop_tokens[i, cur_ppg_len-self.n_frames_per_step:] = 1
|
|
if len(batch[0]) == 5:
|
|
ret_tup = (ppgs_padded, lf0_uvs_padded, mels_padded, torch.LongTensor(ppg_lengths), \
|
|
torch.LongTensor(mel_lengths), spk_ids, stop_tokens)
|
|
if self.give_uttids:
|
|
return ret_tup + (fids, )
|
|
else:
|
|
return ret_tup
|
|
else:
|
|
ret_tup = (ppgs_padded, lf0_uvs_padded, mels_padded, torch.LongTensor(ppg_lengths), \
|
|
torch.LongTensor(mel_lengths), stop_tokens)
|
|
if self.give_uttids:
|
|
return ret_tup + (fids, )
|
|
else:
|
|
return ret_tup
|