{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "broadband-wilson", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "acoustic-ordering", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['CUDA_VISIBLE_DEVICES'] = ''" ] }, { "cell_type": "code", "execution_count": 3, "id": "usual-patrol", "metadata": {}, "outputs": [], "source": [ "import re\n", "import pandas as pd\n", "import numpy as np\n", "import librosa\n", "from scipy.io import wavfile\n", "import librosa.display\n", "from pydub import AudioSegment\n", "from IPython.display import display_html\n", "import torch\n", "import tqdm\n", "\n", "\n", "def _convert_array(arr, force_int=False, force_float=False):\n", " if isinstance(arr, torch.Tensor):\n", " arr = arr.cpu().detach().numpy()\n", " assert(len(arr.shape) in (1, 2))\n", " if len(arr.shape) == 2:\n", " arr = arr.mean(0)\n", " if force_int and \"float\" in str(arr.dtype):\n", " arr = np.floor(arr.copy() * 32768).astype(np.int16)\n", " if force_float and \"int\" in str(arr.dtype):\n", " arr = arr.copy().astype(np.float32) / 32768\n", " return arr\n", "\n", "\n", "def plot_ft(arr, window_size=1024, hop_length=512, n_mels=128):\n", " arr = _convert_array(arr, force_float=True)\n", " window = np.hanning(window_size)\n", " stft= librosa.core.spectrum.stft(arr, n_fft=window_size, hop_length=hop_length, window=window)\n", " out = 2 * np.abs(stft) / np.sum(window)\n", " data = librosa.amplitude_to_db(out, ref=np.max)\n", "\n", " plt.figure(figsize=(16, 10))\n", " librosa.display.specshow(data, x_axis='time', y_axis='mel', sr=16000)\n", " plt.set_cmap('afmhot')\n", "\n", "\n", "def play_array(arr, sr=16000, nc=1):\n", " # peak normalize to avoid clipping\n", " if np.abs(arr).max() >= 1:\n", " arr = arr / (0.1 + np.abs(arr).max())\n", " arr = _convert_array(arr, force_int=True)\n", " display_html(AudioSegment(\n", " arr.tobytes(), \n", " frame_rate=sr,\n", " sample_width=2, \n", " channels=nc\n", " ))" ] }, { "cell_type": "code", "execution_count": 4, "id": "viral-polls", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "################################################################################\n", "### WARNING, path does not exist: KALDI_ROOT=/mnt/matylda5/iveselyk/Tools/kaldi-trunk\n", "### (please add 'export KALDI_ROOT=' in your $HOME/.profile)\n", "### (or run as: KALDI_ROOT= python .py)\n", "################################################################################\n", "\n" ] } ], "source": [ "from espnet2.bin.tts_inference import Text2Speech\n", "import torchaudio\n", "import kaldiio\n", "from espnet2.gan_tts.vits.vits import VITS\n", "from espnet_model_zoo.downloader import ModelDownloader" ] }, { "cell_type": "markdown", "id": "little-scoop", "metadata": {}, "source": [ "# 1) use kan-bayashi/libritts_xvector_vits\n", "\n", "https://github.com/espnet/espnet_model_zoo/blob/master/espnet_model_zoo/table.csv" ] }, { "cell_type": "code", "execution_count": 27, "id": "fixed-friendly", "metadata": {}, "outputs": [], "source": [ "model = Text2Speech.from_pretrained(\n", " model_tag=\"kan-bayashi/libritts_xvector_vits\",\n", " vocoder_tag=None,\n", " device=\"cpu\",\n", " speed_control_alpha=1.0,\n", " noise_scale=0.667,\n", " noise_scale_dur=0.8,\n", ")" ] }, { "cell_type": "code", "execution_count": 23, "id": "integrated-lottery", "metadata": {}, "outputs": [], "source": [ "d = ModelDownloader()\n", "model_dict = d.download_and_unpack(\"kan-bayashi/libritts_xvector_vits\")\n", "\n", "model_dir = os.path.dirname(model_dict[\"train_config\"])\n", "xvector_fp = model_dir + \"/../../dump/22k/xvector/train-clean-460/spk_xvector.ark\"\n", "speaker_xvectors = [v for k, v in kaldiio.load_ark(xvector_fp)]" ] }, { "cell_type": "code", "execution_count": 60, "id": "bound-fleece", "metadata": {}, "outputs": [], "source": [ "model.decode_conf[\"alpha\"] = 0.8\n", "model.decode_conf[\"noise_scale\"] = 0.1\n", "model.decode_conf[\"noise_scale_dur\"] = 0.1" ] }, { "cell_type": "code", "execution_count": 62, "id": "quarterly-piece", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# text = \"Type a sentence here.\"\n", "text = \"And finally, this is it. Am I right? I don't think so.\"\n", "\n", "spk_emb = speaker_xvectors[1]\n", "\n", "with torch.no_grad():\n", " speech = model(text, sids=None, speech=None, spembs=spk_emb)[\"wav\"]\n", " \n", "play_array(speech, sr=model.fs)" ] }, { "cell_type": "markdown", "id": "divine-western", "metadata": {}, "source": [ "# 2) use newest espeak \n", "https://github.com/espnet/espnet/tree/master/egs2/libritts/tts1" ] }, { "cell_type": "code", "execution_count": 43, "id": "closing-cooperative", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ['PATH'] += \":/home/georg/code/espnet/tools/installers/espeak-ng/src\"" ] }, { "cell_type": "code", "execution_count": 44, "id": "israeli-violence", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/georg/venvs/py37/lib/python3.7/site-packages/espnet_model_zoo/downloader.py:218: UserWarning: Expanding a local model to the cachedir. If you'll move the file to another path, it's treated as a different model.\n", " \"Expanding a local model to the cachedir. \"\n" ] } ], "source": [ "model = Text2Speech.from_pretrained(\n", " model_tag=\"/home/georg/models/espnet/tts_train_xvector_vits_raw_phn_tacotron_espeak_ng_english_us_vits_train.total_count.ave.zip\",\n", " vocoder_tag=None,\n", " device=\"cpu\",\n", " speed_control_alpha=1.0,\n", " noise_scale=0.667,\n", " noise_scale_dur=0.8,\n", ")" ] }, { "cell_type": "code", "execution_count": 45, "id": "controversial-vulnerability", "metadata": {}, "outputs": [], "source": [ "xvector_fp = \"/home/georg/models/espnet/tts_train_xvector_vits_raw_phn_tacotron_espeak_ng_english_us_vits_train.total_count.ave/dump/22k/xvector/train-clean-460/spk_xvector.ark\"\n", "speaker_xvectors = [v for k, v in kaldiio.load_ark(xvector_fp)]" ] }, { "cell_type": "code", "execution_count": 46, "id": "operational-replication", "metadata": {}, "outputs": [], "source": [ "model.decode_conf[\"alpha\"] = 0.8\n", "# model.decode_conf[\"noise_scale\"] = 0.1\n", "# model.decode_conf[\"noise_scale_dur\"] = 0.1" ] }, { "cell_type": "code", "execution_count": 49, "id": "virgin-grace", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# text = \"Type a sentence here.\"\n", "# text = \"And finally, this is it. Am I right? I don't think so.\"\n", "text = (\n", " \"And in particular, since 2019, it's worth noting that we've grown the size of the combined \"\n", " \"New Zealand Meridian and Powershop customer bases by 20 percent and the total volume of energy sold \"\n", " \"through our retail channels by 14 percent, while spending less than $13.4.\"\n", ")\n", "\n", "spk_emb = speaker_xvectors[3]\n", "\n", "with torch.no_grad():\n", " speech = model(text, sids=None, speech=None, spembs=spk_emb)[\"wav\"]\n", " \n", "play_array(speech, sr=model.fs)" ] }, { "cell_type": "markdown", "id": "competitive-skating", "metadata": {}, "source": [ "# single speaker" ] }, { "cell_type": "code", "execution_count": 37, "id": "mineral-power", "metadata": {}, "outputs": [], "source": [ "model = Text2Speech.from_pretrained(\n", " model_tag=\"kan-bayashi/ljspeech_vits\",\n", " vocoder_tag=None,\n", " device=\"cpu\",\n", " speed_control_alpha=1.0,\n", " noise_scale=0.667,\n", " noise_scale_dur=0.8,\n", ")" ] }, { "cell_type": "code", "execution_count": 40, "id": "exotic-nickel", "metadata": {}, "outputs": [], "source": [ "model.decode_conf[\"alpha\"] = 0.8\n", "# model.decode_conf[\"noise_scale\"] = 0.1\n", "# model.decode_conf[\"noise_scale_dur\"] = 0.1" ] }, { "cell_type": "code", "execution_count": 42, "id": "prime-silence", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# text = \"Oh my god, this is literally the first time I am hearing about this.\"\n", "# text = \"And finally, this is it. Am I right? I don't think so.\"\n", "text = (\n", " \"And in particular, since 2019, it's worth noting that we've grown the size of the combined \"\n", " \"New Zealand Meridian and Powershop customer bases by 20 percent and the total volume of energy sold \"\n", " \"through our retail channels by 14 percent, while spending less than $13.4.\"\n", ")\n", "\n", "with torch.no_grad():\n", " speech = model(text, sids=None, speech=None, spembs=None)[\"wav\"]\n", " \n", "play_array(speech, sr=model.fs)" ] }, { "cell_type": "code", "execution_count": null, "id": "freelance-stocks", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "pressed-criticism", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bridal-quest", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "partial-elevation", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bigger-reasoning", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "colored-accident", "metadata": {}, "source": [ "# TODO:\n", "latest in we tried was 153 in :\n", "\n", "https://github.com/espnet/espnet_model_zoo/blob/master/espnet_model_zoo/table.csv" ] }, { "cell_type": "code", "execution_count": null, "id": "productive-enforcement", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.12" } }, "nbformat": 4, "nbformat_minor": 5 }