I am adapting to a given voice checkpoint with 400000 steps using my own dataset and it sounds reasonably well in the test sentences.
But when using inference, I often encounter amnesic dysphasia(=“word retrieval failures”): words are being mumbled, audio files end with a hissing sound.
Also high pitched intonations in ground truth are distorted in synthesized speech.
What is the likely cause for this? Please be specific.
What’s the quality of your dataset? What training parameters are you using? Sounds like it’s not aligning well, though why isn’t clear without more information. Do your graphs look correct?
What’s the quality of your dataset?
High quality in origin, converted to 22050Hz,16 bit, WAVE files.
Sentences in metadata.csv contain what is said in the wave files.
What training parameters are you using?
This is config.json of the vocoder that I continue training on.
{
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"dataset/mydataset/pwgan-January-01-2021_08+51AM-49fe63a/best_model.pth.tar",
"github_branch":"* master",
"restore_path":"/start/where/here/myfiles/dataset/mysubfolder/vocoder_model.pth.tar",
"github_branch":"* dev",
"restore_path":"/start/where/here/myfiles/dataset/mydataset/mydataset-ddc-January-01-2021_01+41AM-49fe63a/best_model.pth.tar",
"run_name": "pwgan",
"run_description": "parallel-wavegan for german",
// AUDIO PARAMETERS
"audio":{
// stft parameters
"fft_size": 1024, // number of stft frequency levels. Size of the linear spectogram frame.
"win_length": 1024, // stft window length in ms.
"hop_length": 256, // stft window hop-lengh in ms.
"frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used.
"frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used.
// Audio processing parameters
"sample_rate": 22050, // DATASET-RELATED: wav sample-rate.
"preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis.
"ref_level_db": 20, // reference level db, theoretically 20db is the sound of air.
// Silence trimming
"do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (true), TWEB (false), Nancy (true)
"trim_db": 60, // threshold for timming silence. Set this according to your dataset.
"do_sound_norm": true,
// Griffin-Lim
"power": 1.5, // value to sharpen wav signals after GL algorithm.
"griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation.
// MelSpectrogram parameters
"num_mels": 80, // size of the mel spec frame.
"mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!!
"mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!!
"spec_gain": 20.0,
// Normalization parameters
"signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params.
"min_level_db": -100, // lower bound for normalization
"symmetric_norm": true, // move normalization to range [-1, 1]
"max_norm": 1.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm]
"clip_norm": true, // clip normalized values into the range.
"stats_path": "/start/where/here/myfiles/dataset/mydataset/scale_stats.npy" // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored
},
// DISTRIBUTED TRAINING
// "distributed":{
// "backend": "nccl",
// "url": "tcp:\/\/localhost:54321"
// },
// MODEL PARAMETERS
"use_pqmf": false,
// LOSS PARAMETERS
"use_stft_loss": true,
"use_subband_stft_loss": false, // USE ONLY WITH MULTIBAND MODELS
"use_mse_gan_loss": true,
"use_hinge_gan_loss": false,
"use_feat_match_loss": false, // use only with melgan discriminators
// loss weights
"stft_loss_weight": 0.5,
"subband_stft_loss_weight": 0.5,
"mse_G_loss_weight": 2.5,
"hinge_G_loss_weight": 2.5,
"feat_match_loss_weight": 25,
// multiscale stft loss parameters
"stft_loss_params": {
"n_ffts": [1024, 2048, 512],
"hop_lengths": [120, 240, 50],
"win_lengths": [600, 1200, 240]
},
// subband multiscale stft loss parameters
"subband_stft_loss_params":{
"n_ffts": [384, 683, 171],
"hop_lengths": [30, 60, 10],
"win_lengths": [150, 300, 60]
},
"target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch
// DISCRIMINATOR
"discriminator_model": "parallel_wavegan_discriminator",
"discriminator_model_params":{
"num_layers": 10
},
"steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1
// GENERATOR
"generator_model": "parallel_wavegan_generator",
"generator_model_params": {
"upsample_factors":[4, 4, 4, 4],
"stacks": 3,
"num_res_blocks": 30,
"aux_context_window": 0
},
// DATASET
"data_path": "/start/where/here/myfiles/dataset/mydataset/wavs/",
"feature_path": null,
"seq_len": 25600,
"pad_short": 2000,
"conv_pad": 0,
"use_noise_augment": false,
"use_cache": true,
"reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers.
// TRAINING
"batch_size": 6, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'.
// VALIDATION
"run_eval": true,
"test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time.
"test_sentences_file": "/start/where/here/myfiles/dataset/mysubfolder/de_sentences.txt", // set a file to load sentences to be used for testing. If it is null then we use default english sentences.
// OPTIMIZER
"epochs": 10000, // total number of epochs to train.
"wd": 0.0, // Weight decay weight.
"gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0
"disc_clip_grad": -1, // Discriminator gradient clipping threshold.
"lr_scheduler_gen": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
"lr_scheduler_gen_params": {
"gamma": 0.5,
"milestones": [100000, 200000, 300000, 400000, 500000, 600000]
},
"lr_scheduler_disc": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
"lr_scheduler_disc_params": {
"gamma": 0.5,
"milestones": [100000, 200000, 300000, 400000, 500000, 600000]
},
"lr_gen": 1e-4, // Initial learning rate. If Noam decay is active, maximum learning rate.
"lr_disc": 1e-4,
// TENSORBOARD and LOGGING
"print_step": 25, // Number of steps to log traning on console.
"print_eval": false, // If True, it prints loss values for each step in eval run.
"save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints.
"checkpoint": true, // If true, it saves checkpoints per "save_step"
"tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging.
// DATA LOADING
"num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values.
"num_val_loader_workers": 4, // number of evaluation data loader processes.
"eval_split_size": 10,
// PATHS
"output_path": "/start/where/here/myfiles/dataset/mydataset/"
}
Do your graphs look correct?
Any ideas what is wrong with this?
This is probably going to be fairly difficult to diagnose but it would be useful to see the charts for the TTS model (you appear to have posted just charts from the vocoder side which will likely have less influence on the kind of issues you’re reporting)
