some minor fixes

deployment/exporters/variance_exporter.py

@@ -691,7 +691,7 @@ def _optimize_merge_pitch_predictor_graph(
         assert check, 'Simplified ONNX model could not be validated'
 
         onnx_helper.model_override_io_shapes(
-            pitch_predictor, output_shapes={'pitch_pred': (1, 'n_frames')}
+            pitch_predictor, output_shapes={'x_pred': (1, 'n_frames')}
         )
         print(f'Running ONNX Simplifier #1 on {self.pitch_predictor_class_name}...')
         pitch_predictor, check = onnxsim.simplify(pitch_predictor, include_subgraph=True)

inference/ds_variance.py

@@ -241,14 +241,15 @@ def preprocess_input(
         batch['midi'] = ph_midi
 
         if load_pitch:
+            # Interpolate unvoiced parts before resampling.
             f0 = resample_align_curve(
-                np.array(param['f0_seq'].split(), np.float32),
+                interp_f0(np.array(param['f0_seq'].split(), np.float32))[0],
                 original_timestep=float(param['f0_timestep']),
                 target_timestep=self.timestep,
                 align_length=T_s
             )
             batch['pitch'] = torch.from_numpy(
-                librosa.hz_to_midi(interp_f0(f0)[0]).astype(np.float32)
+                librosa.hz_to_midi(f0).astype(np.float32)
             ).to(self.device)[None]
 
         if self.model.predict_dur:
@@ -442,19 +443,18 @@ def run_inference(
                     param_copy[f'{v_name}_timestep'] = str(self.timestep)
 
                 # Restore ph_spk_mix and spk_mix
-                if 'ph_spk_mix' in param_copy and 'spk_mix' in param_copy:
-                    if 'ph_spk_mix_backup' in param_copy:
-                        if param_copy['ph_spk_mix_backup'] is None:
-                            del param_copy['ph_spk_mix']
-                        else:
-                            param_copy['ph_spk_mix'] = param_copy['ph_spk_mix_backup']
-                        del param['ph_spk_mix_backup']
-                    if 'spk_mix_backup' in param_copy:
-                        if param_copy['ph_spk_mix_backup'] is None:
-                            del param_copy['spk_mix']
-                        else:
-                            param_copy['spk_mix'] = param_copy['spk_mix_backup']
-                        del param['spk_mix_backup']
+                if 'ph_spk_mix_backup' in param_copy:
+                    if param_copy['ph_spk_mix_backup'] is None:
+                        param_copy.pop('ph_spk_mix', None)
+                    else:
+                        param_copy['ph_spk_mix'] = param_copy['ph_spk_mix_backup']
+                    del param_copy['ph_spk_mix_backup']
+                if 'spk_mix_backup' in param_copy:
+                    if param_copy['spk_mix_backup'] is None:
+                        param_copy.pop('spk_mix', None)
+                    else:
+                        param_copy['spk_mix'] = param_copy['spk_mix_backup']
+                    del param_copy['spk_mix_backup']
 
                 results.append(param_copy)
 

modules/toplevel.py

@@ -341,7 +341,10 @@ def forward(
             return dur_pred_out, pitch_pred_out, ({} if infer else None)
 
         if pitch is None:
-            pitch = base_pitch + pitch_pred_out
+            if pitch_pred_out is not None:
+                pitch = base_pitch + pitch_pred_out
+            else:
+                pitch = base_pitch
         if self.use_variance_scaling:
             var_cond = condition + self.pitch_embed(pitch[:, :, None] / 12)
         else:

preprocessing/acoustic_binarizer.py

@@ -319,7 +319,8 @@ def arrange_data_augmentation(self, data_iterator):
             k_from_aug = int(total_scale * scale / (1 + total_scale) * len(all_item_names))
             k_mutate = int(total_scale * scale / (1 + scale) * len(all_item_names))
             aug_types = [0] * k_from_raw + [1] * k_from_aug + [2] * k_mutate
-            aug_items = random.choices(all_item_names, k=k_from_raw) + random.choices(aug_list, k=k_from_aug + k_mutate)
+            aug_items = random.choices(all_item_names, k=k_from_raw) + \
+                random.choices(aug_list, k=k_from_aug) + random.sample(aug_list, k=min(k_mutate, len(aug_list)))
 
             for aug_type, aug_item in zip(aug_types, aug_items):
                 # Uniform distribution in log domain
@@ -337,7 +338,7 @@ def arrange_data_augmentation(self, data_iterator):
                     aug_list.append(aug_task)
                 elif aug_type == 1:
                     aug_task = {
-                        'name': aug_item,
+                        'name': aug_item['name'],
                         'func': aug_item['func'],
                         'kwargs': deepcopy(aug_item['kwargs'])
                     }

preprocessing/variance_binarizer.py

@@ -107,7 +107,7 @@ def load_attr_from_ds(self, ds_id, name, attr, idx=0):
             if not isinstance(ds, list):
                 ds = [ds]
             self.cached_ds[cache_key] = ds
-            ds = ds[idx]
+            ds = ds[0] if cache_key == item_name_with_idx else ds[idx]
         return ds.get(attr)
 
     def load_meta_data(self, raw_data_dir: pathlib.Path, ds_id, spk, lang):
@@ -117,11 +117,12 @@ def load_meta_data(self, raw_data_dir: pathlib.Path, ds_id, spk, lang):
             for utterance_label in csv.DictReader(f):
                 utterance_label: dict
                 item_name = utterance_label['name']
-                item_idx = int(item_name.rsplit(DS_INDEX_SEP, maxsplit=1)[-1]) if DS_INDEX_SEP in item_name else 0
+                item_base_name, *item_seg_idx = item_name.rsplit(DS_INDEX_SEP, maxsplit=1)
+                item_idx = int(item_seg_idx[0]) if item_seg_idx else 0
 
                 def require(attr, optional=False):
                     if self.prefer_ds:
-                        value = self.load_attr_from_ds(ds_id, item_name, attr, item_idx)
+                        value = self.load_attr_from_ds(ds_id, item_base_name, attr, item_idx)
                     else:
                         value = None
                     if value is None:
@@ -275,15 +276,13 @@ def process_item(self, item_name, meta_data, binarization_args):
         ds_id = int(ds_id)
         ds_seg_idx = meta_data['ds_idx']
         seconds = sum(meta_data['ph_dur'])
-        length = round(seconds / self.timestep)
         T_ph = len(meta_data['ph_seq'])
         processed_input = {
             'name': item_name,
             'wav_fn': meta_data['wav_fn'],
             'spk_id': meta_data['spk_id'],
             'spk_name': meta_data['spk_name'],
             'seconds': seconds,
-            'length': length,
             'languages': np.array(meta_data['lang_seq'], dtype=np.int64),
             'tokens': np.array(meta_data['ph_seq'], dtype=np.int64),
             'ph_text': meta_data['ph_text'],
@@ -292,7 +291,9 @@ def process_item(self, item_name, meta_data, binarization_args):
         ph_dur_sec = torch.FloatTensor(meta_data['ph_dur']).to(self.device)
         ph_acc = torch.round(torch.cumsum(ph_dur_sec, dim=0) / self.timestep + 0.5).long()
         ph_dur = torch.diff(ph_acc, dim=0, prepend=torch.LongTensor([0]).to(self.device))
+        length = int(ph_acc[-1])
         processed_input['ph_dur'] = ph_dur.cpu().numpy()
+        processed_input['length'] = length
 
         mel2ph = get_mel2ph_torch(
             self.lr, ph_dur_sec, length, self.timestep, device=self.device
@@ -314,14 +315,21 @@ def process_item(self, item_name, meta_data, binarization_args):
         if self.prefer_ds:
             f0_seq = self.load_attr_from_ds(ds_id, name, 'f0_seq', idx=ds_seg_idx)
             if f0_seq is not None:
+                f0_timestep = float(self.load_attr_from_ds(ds_id, name, 'f0_timestep', idx=ds_seg_idx))
+                # Interpolate unvoiced parts before resampling.
+                f0_points, uv_points = interp_f0(np.array(f0_seq.split(), np.float32))
                 f0 = resample_align_curve(
-                    np.array(f0_seq.split(), np.float32),
-                    original_timestep=float(self.load_attr_from_ds(ds_id, name, 'f0_timestep', idx=ds_seg_idx)),
+                    f0_points,
+                    original_timestep=f0_timestep,
                     target_timestep=self.timestep,
                     align_length=length
                 )
-                uv = f0 == 0
-                f0, _ = interp_f0(f0, uv)
+                uv = resample_align_curve(
+                    uv_points.astype(np.float32),
+                    original_timestep=f0_timestep,
+                    target_timestep=self.timestep,
+                    align_length=length
+                ) > 0.5
         if f0 is None:
             f0, uv = pitch_extractor.get_pitch(
                 waveform, samplerate=hparams['audio_sample_rate'], length=length,

utils/decomposed_waveform.py

@@ -75,7 +75,7 @@ def _init(
         # extraction parameters
         self._hop_size = hop_size
         self._fft_size = fft_size if fft_size is not None else win_size
-        self._win_size = win_size if win_size is not None else win_size
+        self._win_size = win_size if win_size is not None else fft_size
         self._time_step = hop_size / samplerate
         self._half_width = base_harmonic_radius
         self._device = ('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device

utils/infer_utils.py

@@ -39,17 +39,11 @@ def trans_key(raw_data, key):
 
 
 def resample_align_curve(points: np.ndarray, original_timestep: float, target_timestep: float, align_length: int):
-    t_max = (len(points) - 1) * original_timestep
     curve_interp = np.interp(
-        np.arange(0, t_max, target_timestep),
+        np.arange(align_length) * target_timestep,
         original_timestep * np.arange(len(points)),
         points
     ).astype(points.dtype)
-    delta_l = align_length - len(curve_interp)
-    if delta_l < 0:
-        curve_interp = curve_interp[:align_length]
-    elif delta_l > 0:
-        curve_interp = np.concatenate((curve_interp, np.full(delta_l, fill_value=curve_interp[-1])), axis=0)
     return curve_interp
 
 

utils/onnx_helper.py

@@ -80,7 +80,7 @@ def model_reorder_io_list(
     :param model: model to perform the operation on
     :param input_or_output: 'input' or 'output' to specify the list to reorder
     :param target_name: the name of the input to be reordered
-    :param insert_after_name: the name of the input to be inserted after (None for the first)
+    :param insert_after_name: the name of the input to be inserted after
     """
     def _reorder_input(input_list: RepeatedCompositeFieldContainer[ValueInfoProto]):
         nonlocal input_or_output
@@ -93,7 +93,8 @@ def _reorder_input(input_list: RepeatedCompositeFieldContainer[ValueInfoProto]):
                 insert_after_idx = i
         if target_idx != -1 and insert_after_idx != -1:
             target = input_list.pop(target_idx)
-            input_list.insert(insert_after_idx + 1, target)
+            insert_at = insert_after_idx + (1 if target_idx > insert_after_idx else 0)
+            input_list.insert(insert_at, target)
             _verbose(f'| reorder {input_or_output}: \'{target_name}\' after \'{insert_after_name}\'')
 
     if input_or_output == 'input':