Remove obsolete package file tiny_tts/infer_onnx.py
Browse files- tiny_tts/infer_onnx.py +0 -199
tiny_tts/infer_onnx.py
DELETED
|
@@ -1,199 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
ONNX Runtime inference engine for TinyTTS.
|
| 3 |
-
|
| 4 |
-
Replaces the PyTorch VoiceSynthesizer.infer() with equivalent
|
| 5 |
-
ONNX Runtime sessions + NumPy ops for the non-exported parts
|
| 6 |
-
(alignment path computation).
|
| 7 |
-
"""
|
| 8 |
-
import os
|
| 9 |
-
import numpy as np
|
| 10 |
-
import soundfile as sf
|
| 11 |
-
|
| 12 |
-
from tiny_tts.text.english import normalize_text, grapheme_to_phoneme
|
| 13 |
-
from tiny_tts.text import phonemes_to_ids
|
| 14 |
-
from tiny_tts.nn import commons
|
| 15 |
-
from tiny_tts.utils.config import (
|
| 16 |
-
SAMPLING_RATE, ADD_BLANK, SPK2ID,
|
| 17 |
-
)
|
| 18 |
-
|
| 19 |
-
try:
|
| 20 |
-
import onnxruntime as ort
|
| 21 |
-
except ImportError:
|
| 22 |
-
raise ImportError("onnxruntime is required. Run: pip install onnxruntime")
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
def _build_session(path: str, use_gpu: bool = False):
|
| 26 |
-
"""Create an ORT InferenceSession with optional GPU support."""
|
| 27 |
-
providers = (
|
| 28 |
-
["CUDAExecutionProvider", "CPUExecutionProvider"]
|
| 29 |
-
if use_gpu else
|
| 30 |
-
["CPUExecutionProvider"]
|
| 31 |
-
)
|
| 32 |
-
opts = ort.SessionOptions()
|
| 33 |
-
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 34 |
-
opts.intra_op_num_threads = os.cpu_count() or 4
|
| 35 |
-
return ort.InferenceSession(path, sess_options=opts, providers=providers)
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
def _create_length_mask_np(lengths, max_len=None):
|
| 39 |
-
"""NumPy equivalent of commons.create_length_mask."""
|
| 40 |
-
if max_len is None:
|
| 41 |
-
max_len = int(lengths.max())
|
| 42 |
-
ids = np.arange(max_len, dtype=np.float32) # [T]
|
| 43 |
-
mask = (ids[None, :] < lengths[:, None]).astype(np.float32) # [B, T]
|
| 44 |
-
return mask
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def _compute_alignment_path_np(w_ceil, attn_mask):
|
| 48 |
-
"""
|
| 49 |
-
Monotonic alignment path - vectorized via cumsum (much faster than Python loops).
|
| 50 |
-
w_ceil: [B, 1, T_x] — integer duration per phone
|
| 51 |
-
attn_mask: [B, 1, T_y, T_x] — joint mask
|
| 52 |
-
Returns attn: [B, 1, T_y, T_x]
|
| 53 |
-
"""
|
| 54 |
-
B, _, T_x = w_ceil.shape
|
| 55 |
-
T_y = attn_mask.shape[2]
|
| 56 |
-
|
| 57 |
-
# Build duration matrix: for each phone column expand the duration
|
| 58 |
-
# cumulative sum of durations gives us the end frame index for each phone
|
| 59 |
-
dur = w_ceil[:, 0, :] # [B, T_x]
|
| 60 |
-
cum_dur = np.cumsum(dur, axis=1) # [B, T_x] — end frame (1-indexed)
|
| 61 |
-
cum_dur_prev = np.pad(cum_dur[:, :-1], ((0,0),(1,0))) # [B, T_x] — start frame
|
| 62 |
-
|
| 63 |
-
# Frame indices: [1, T_y, 1]
|
| 64 |
-
frame_idx = np.arange(T_y, dtype=np.float32)[None, :, None] # [1, T_y, 1]
|
| 65 |
-
# For each phone, mark frames [start, end)
|
| 66 |
-
# cum_dur_prev: [B,1,T_x], cum_dur: [B,1,T_x]
|
| 67 |
-
start = cum_dur_prev[:, None, :] # [B, 1, T_x]
|
| 68 |
-
end = cum_dur[:, None, :] # [B, 1, T_x]
|
| 69 |
-
attn = ((frame_idx >= start) & (frame_idx < end)).astype(np.float32) # [B, T_y, T_x]
|
| 70 |
-
attn = attn[:, None, :, :] # [B, 1, T_y, T_x]
|
| 71 |
-
return attn * attn_mask
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
class OnnxTinyTTS:
|
| 75 |
-
"""
|
| 76 |
-
Inference using ONNX Runtime.
|
| 77 |
-
|
| 78 |
-
Args:
|
| 79 |
-
onnx_dir: directory containing the 4 .onnx files
|
| 80 |
-
use_gpu: if True, try CUDAExecutionProvider
|
| 81 |
-
"""
|
| 82 |
-
|
| 83 |
-
def __init__(self, onnx_dir: str = "onnx", use_gpu: bool = False):
|
| 84 |
-
onnx_dir = os.path.abspath(onnx_dir)
|
| 85 |
-
print(f"Loading ONNX sessions from: {onnx_dir}")
|
| 86 |
-
|
| 87 |
-
self._enc = _build_session(os.path.join(onnx_dir, "text_encoder.onnx"), use_gpu)
|
| 88 |
-
self._dp = _build_session(os.path.join(onnx_dir, "duration_predictor.onnx"), use_gpu)
|
| 89 |
-
self._flow = _build_session(os.path.join(onnx_dir, "flow.onnx"), use_gpu)
|
| 90 |
-
self._dec = _build_session(os.path.join(onnx_dir, "decoder.onnx"), use_gpu)
|
| 91 |
-
|
| 92 |
-
print("ONNX sessions ready ✅")
|
| 93 |
-
|
| 94 |
-
def _text_to_ids(self, text: str):
|
| 95 |
-
normalized = normalize_text(text)
|
| 96 |
-
phones, tones, _ = grapheme_to_phoneme(normalized)
|
| 97 |
-
phone_ids, tone_ids, lang_ids = phonemes_to_ids(phones, tones, "EN")
|
| 98 |
-
|
| 99 |
-
if ADD_BLANK:
|
| 100 |
-
phone_ids = commons.insert_blanks(phone_ids, 0)
|
| 101 |
-
tone_ids = commons.insert_blanks(tone_ids, 0)
|
| 102 |
-
lang_ids = commons.insert_blanks(lang_ids, 0)
|
| 103 |
-
|
| 104 |
-
return phone_ids, tone_ids, lang_ids
|
| 105 |
-
|
| 106 |
-
def speak(
|
| 107 |
-
self,
|
| 108 |
-
text: str,
|
| 109 |
-
output_path: str = "onnx_output.wav",
|
| 110 |
-
speaker: str = "female",
|
| 111 |
-
noise_scale: float = 0.667,
|
| 112 |
-
noise_scale_w: float = 0.8,
|
| 113 |
-
length_scale: float = 1.0,
|
| 114 |
-
output_sr: int = None,
|
| 115 |
-
) -> np.ndarray:
|
| 116 |
-
"""Synthesize speech and save to output_path.
|
| 117 |
-
|
| 118 |
-
Args:
|
| 119 |
-
output_sr: If set (e.g. 22050), resample the output from 44100 Hz.
|
| 120 |
-
Useful to reduce file size while keeping quality.
|
| 121 |
-
"""
|
| 122 |
-
print(f"[ONNX] Synthesizing: {text}")
|
| 123 |
-
|
| 124 |
-
phone_ids, tone_ids, lang_ids = self._text_to_ids(text)
|
| 125 |
-
T = len(phone_ids)
|
| 126 |
-
|
| 127 |
-
# Prepare inputs as float32 / int64 arrays
|
| 128 |
-
x = np.array(phone_ids, dtype=np.int64)[None, :] # [1, T]
|
| 129 |
-
x_len = np.array([T], dtype=np.int64) # [1]
|
| 130 |
-
tone = np.array(tone_ids, dtype=np.int64)[None, :] # [1, T]
|
| 131 |
-
lang = np.array(lang_ids, dtype=np.int64)[None, :] # [1, T]
|
| 132 |
-
bert = np.zeros((1, 1024, T), dtype=np.float32)
|
| 133 |
-
ja_bert = np.zeros((1, 768, T), dtype=np.float32)
|
| 134 |
-
sid_val = SPK2ID.get(speaker, 0)
|
| 135 |
-
sid = np.array([sid_val], dtype=np.int64) # [1]
|
| 136 |
-
|
| 137 |
-
# ── 1. Text Encoder ──────────────────────────────────────────────
|
| 138 |
-
x_enc, m_p, logs_p, x_mask, g = self._enc.run(
|
| 139 |
-
None,
|
| 140 |
-
{
|
| 141 |
-
"phone_ids": x,
|
| 142 |
-
"phone_lengths":x_len,
|
| 143 |
-
"tone_ids": tone,
|
| 144 |
-
"language_ids": lang,
|
| 145 |
-
"bert": bert,
|
| 146 |
-
"ja_bert": ja_bert,
|
| 147 |
-
"speaker_id": sid,
|
| 148 |
-
},
|
| 149 |
-
)
|
| 150 |
-
|
| 151 |
-
# ── 2. Duration Predictor ─────────────────────────────────────────
|
| 152 |
-
logw = self._dp.run(None, {"x": x_enc, "x_mask": x_mask, "g": g})[0]
|
| 153 |
-
|
| 154 |
-
# ── 3. Alignment Path (NumPy) ─────────────────────────────────────
|
| 155 |
-
w = np.exp(logw) * x_mask * length_scale # [1, 1, T]
|
| 156 |
-
w_ceil = np.ceil(w) # [1, 1, T]
|
| 157 |
-
y_len = max(1, int(w_ceil.sum()))
|
| 158 |
-
y_lens = np.array([y_len], dtype=np.int64)
|
| 159 |
-
|
| 160 |
-
y_mask = _create_length_mask_np(y_lens, y_len) # [1, T_y]
|
| 161 |
-
y_mask = y_mask[:, None, :] # [1, 1, T_y]
|
| 162 |
-
# attn_mask: [1, 1, T_y, T_x] (outer product of frame mask and phone mask)
|
| 163 |
-
attn_mask = y_mask[:, :, :, None] * x_mask[:, :, None, :] # [1,1,T_y,T_x]
|
| 164 |
-
attn = _compute_alignment_path_np(w_ceil, attn_mask) # [1, 1, T_y, T_x]
|
| 165 |
-
|
| 166 |
-
# Expand prior stats via alignment
|
| 167 |
-
m_p_exp = np.matmul(attn[:, 0], m_p.transpose(0, 2, 1)).transpose(0, 2, 1)
|
| 168 |
-
logs_p_exp = np.matmul(attn[:, 0], logs_p.transpose(0, 2, 1)).transpose(0, 2, 1)
|
| 169 |
-
|
| 170 |
-
# ── 4. Sample z_p ─────────────────────────────────────────────────
|
| 171 |
-
z_p = m_p_exp + np.random.randn(*m_p_exp.shape).astype(np.float32) * \
|
| 172 |
-
np.exp(logs_p_exp) * noise_scale
|
| 173 |
-
|
| 174 |
-
# ── 5. Flow (reverse) ─────────────────────────────────────────────
|
| 175 |
-
z = self._flow.run(
|
| 176 |
-
None,
|
| 177 |
-
{"z_p": z_p, "y_mask": y_mask.astype(np.float32), "g": g},
|
| 178 |
-
)[0]
|
| 179 |
-
|
| 180 |
-
# ── 6. Decoder ────────────────────────────────────────────────────
|
| 181 |
-
z_masked = (z * y_mask).astype(np.float32)
|
| 182 |
-
audio = self._dec.run(None, {"z": z_masked, "g": g})[0] # [1, 1, samples]
|
| 183 |
-
|
| 184 |
-
audio_np = audio[0, 0]
|
| 185 |
-
save_sr = SAMPLING_RATE
|
| 186 |
-
if output_sr is not None and output_sr != SAMPLING_RATE:
|
| 187 |
-
try:
|
| 188 |
-
import torchaudio
|
| 189 |
-
import torch
|
| 190 |
-
wav_t = torch.from_numpy(audio_np).unsqueeze(0)
|
| 191 |
-
resampler = torchaudio.transforms.Resample(SAMPLING_RATE, output_sr)
|
| 192 |
-
audio_np = resampler(wav_t).squeeze(0).numpy()
|
| 193 |
-
save_sr = output_sr
|
| 194 |
-
except Exception as e:
|
| 195 |
-
print(f"[ONNX] Resampling failed ({e}), saving at {SAMPLING_RATE}Hz")
|
| 196 |
-
|
| 197 |
-
sf.write(output_path, audio_np, save_sr)
|
| 198 |
-
print(f"[ONNX] Saved: {output_path} ({save_sr}Hz)")
|
| 199 |
-
return audio_np
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|