{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "43b6c065", "metadata": {}, "outputs": [], "source": [ "# https://github.com/SpeechColab/GigaSpeech\n", "\n", "# {\n", "# \"dataset\": \"GigaSpeech\",\n", "# \"language\": \"EN\",\n", "# \"version\": \"v1.0.0\",\n", "# ... ...\n", "# \"audios\": [\n", "# {\n", "# \"title\": \"The Architect of Hollywood\",\n", "# \"url\": \"https://99percentinvisible.org/episode/the-architect-of-hollywood/download\",\n", "# \"path\": \"audio/podcast/P0001/POD0000000025.opus\",\n", "# ... ...\n", "# \"segments\": [\n", "# {\n", "# \"sid\": \"POD0000000025_S0000103\",\n", "# \"speaker\": \"N/A\",\n", "# \"begin_time\": 780.31,\n", "# \"end_time\": 783.13,\n", "# \"text_tn\": \"FOUR O'CLOCK TOMORROW AFTERNOON SAID WILLIAMS \",\n", "# \"subsets\": [\n", "# \"{XL}\",\n", "# \"{L}\"\n", "# ]\n", "# },\n", "# ... ...\n", "# ],\n", "# ... ...\n", "# },\n", "# ... ...\n", "# ]\n", "# }" ] }, { "cell_type": "code", "execution_count": 1, "id": "6a4eda9e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 121, "id": "d457546e", "metadata": {}, "outputs": [], "source": [ "import re\n", "import pandas as pd\n", "import numpy as np\n", "import librosa\n", "from scipy.io import wavfile\n", "import librosa.display\n", "from pydub import AudioSegment\n", "from IPython.display import display_html\n", "import torch\n", "\n", "\n", "def _array_to_numpy(arr):\n", " # convert to np if torch\n", " if isinstance(arr, torch.Tensor):\n", " arr = arr.detach().cpu().numpy()\n", " # squash extra dimensions\n", " arr = arr.squeeze()\n", " # TODO: deal with 2 channels\n", " if 2 in arr.shape:\n", " raise NotImplementedError(\"stereo detected\")\n", " return arr\n", " \n", " \n", "def _array_to_float(arr):\n", " arr = _array_to_numpy(arr)\n", " arr = arr = arr.copy().astype(np.float32) / 32768\n", " return arr\n", "\n", "\n", "def _array_to_int(arr):\n", " arr = _array_to_numpy(arr)\n", " # enforce int16\n", " if arr.dtype == np.int16:\n", " pass\n", " elif arr.dtype in (np.float32, np.float64):\n", " # alert if signal too high\n", " if np.abs(arr).max() > 1:\n", " raise ValueError(\"signal overflow\")\n", " arr = (arr * 32768).clip(-32768, 32767).astype(np.int16)\n", " else:\n", " # TODO: convert other formats\n", " raise NotImplementedError(\"unknown format\")\n", " # alert if audio too loud\n", " if np.abs(arr).mean() / 32768 >= 0.1:\n", " raise ValueError(\"audio seems too loud\")\n", " return arr\n", "\n", "\n", "def play_array(arr, sr=16000): \n", " arr = _array_to_int(arr)\n", " display(AudioSegment(\n", " arr.tobytes(), \n", " frame_rate=sr,\n", " sample_width=2, \n", " channels=1\n", " ))" ] }, { "cell_type": "code", "execution_count": 7, "id": "4a3961ff", "metadata": {}, "outputs": [], "source": [ "DATA_DIR = \"/mnt/data-ssd-1/data/gigaspeech/\"" ] }, { "cell_type": "code", "execution_count": 4, "id": "7ed05267", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "audio GigaSpeech.json\tGigaSpeech.json.gz.aes\tTERMS_OF_ACCESS\r\n" ] } ], "source": [ "!ls /mnt/data-ssd-1/data/gigaspeech" ] }, { "cell_type": "code", "execution_count": 5, "id": "bead851f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[RESEARCHER_FULLNAME] (the \"Researcher\") has requested permission to use the GigaSpeech database (the \"Database\") at Tsinghua University. In exchange for such permission, Researcher hereby agrees to the following terms and conditions:\r\n", "\r\n", "1. Researcher shall use the Database only for non-commercial research and educational purposes.\r\n", "2. The SpeechColab team and Tsinghua University make no representations or warranties regarding the Database, including but not limited to warranties of non-infringement or fitness for a particular purpose.\r\n", "3. Researcher accepts full responsibility for his or her use of the Database and shall defend and indemnify the SpeechColab team and Tsinghua University, including their employees, Trustees, officers and agents, against any and all claims arising from Researcher's use of the Database, including but not limited to Researcher's use of any copies of copyrighted audio files that he or she may create from the Database.\r\n", "4. Researcher may provide research associates and colleagues with access to the Database provided that they first agree to be bound by these terms and conditions.\r\n", "5. The SpeechColab team and Tsinghua University reserve the right to terminate Researcher's access to the Database at any time.\r\n", "6. If Researcher is employed by a for-profit, commercial entity, Researcher's employer shall also be bound by these terms and conditions, and Researcher hereby represents that he or she is fully authorized to enter into this agreement on behalf of such employer.\r\n" ] } ], "source": [ "!cat /mnt/data-ssd-1/data/gigaspeech/TERMS_OF_ACCESS" ] }, { "cell_type": "code", "execution_count": 17, "id": "6ca94f2a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
titleurlpathaidsourceformatsample_ratecodecchannelsmd5durationsizespeakercategorysubsetssegments
0Check Cashing Storeshttps://99percentinvisible.org/episode/episode...audio/podcast/P0001/POD0000000001.opusPOD0000000001podcastopus16000s16le1a1eb19f5bc68d4af71f65a3e8d700343629.9312555888N/AArts[{XL}, {L}][{'sid': 'POD0000000001_S0000008', 'speaker': ...
1Cold War Kidshttps://99percentinvisible.org/episode/cold-wa...audio/podcast/P0001/POD0000000002.opusPOD0000000002podcastopus16000s16le122aff12d1e852338593a6393ceb626041735.5117145217N/AArts[{XL}][{'sid': 'POD0000000002_S0000004', 'speaker': ...
\n", "
" ], "text/plain": [ " title url \\\n", "0 Check Cashing Stores https://99percentinvisible.org/episode/episode... \n", "1 Cold War Kids https://99percentinvisible.org/episode/cold-wa... \n", "\n", " path aid source format \\\n", "0 audio/podcast/P0001/POD0000000001.opus POD0000000001 podcast opus \n", "1 audio/podcast/P0001/POD0000000002.opus POD0000000002 podcast opus \n", "\n", " sample_rate codec channels md5 duration \\\n", "0 16000 s16le 1 a1eb19f5bc68d4af71f65a3e8d700343 629.931 \n", "1 16000 s16le 1 22aff12d1e852338593a6393ceb62604 1735.511 \n", "\n", " size speaker category subsets \\\n", "0 2555888 N/A Arts [{XL}, {L}] \n", "1 7145217 N/A Arts [{XL}] \n", "\n", " segments \n", "0 [{'sid': 'POD0000000001_S0000008', 'speaker': ... \n", "1 [{'sid': 'POD0000000002_S0000004', 'speaker': ... " ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import json\n", "with open(DATA_DIR + \"GigaSpeech.json\") as f:\n", " data = json.load(f)\n", "df = pd.DataFrame(data[\"audios\"])\n", "del data\n", "df.head(2)" ] }, { "cell_type": "code", "execution_count": 110, "id": "effa521f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "10.0 minutes\n", "\n", "{'sid': 'POD0000000017_S0000003', 'speaker': 'N/A', 'begin_time': 148.25, 'end_time': 150.41, 'text_raw': '', 'text_tn': 'THIS IS NINETY NINE PERCENT INVISIBLE ', 'subsets': ['{XL}']}\n" ] }, { "data": { "text/plain": [ "title Penn Station Sucks\n", "url https://99percentinvisible.org/episode/penn-st...\n", "path audio/podcast/P0001/POD0000000017.opus\n", "aid POD0000000017\n", "source podcast\n", "format opus\n", "sample_rate 16000\n", "codec s16le\n", "channels 1\n", "md5 2182ee7c54c7358cac499d27d64dcf5e\n", "duration 1427.292\n", "size 5829184\n", "speaker N/A\n", "category Arts\n", "subsets [{XL}, {L}]\n", "segments [{'sid': 'POD0000000017_S0000003', 'speaker': ...\n", "Name: 16, dtype: object" ] }, "execution_count": 110, "metadata": {}, "output_type": "execute_result" } ], "source": [ "n = 16\n", "row = df[df['url'].str.contains('99percent')].iloc[n]\n", "print(round(sum([d['end_time'] - d['begin_time'] for d in row['segments']]) / 60, 1), 'minutes')\n", "print()\n", "print(row['segments'][0])\n", "row" ] }, { "cell_type": "code", "execution_count": 52, "id": "96798d26", "metadata": {}, "outputs": [], "source": [ "audio = AudioSegment.from_file(DATA_DIR + row['path'], codec='opus')" ] }, { "cell_type": "code", "execution_count": 115, "id": "2c5182c9", "metadata": {}, "outputs": [], "source": [ "n_space_ms = 500\n", "sample_rate = audio_segment.frame_rate\n", "space_arr = np.zeros(int(sample_rate * n_space_ms / 1000), dtype=np.int16)\n", "\n", "data = []\n", "meta = []\n", "offset_frames = 0\n", "for segment in row['segments']:\n", " start_ms = round(segment['begin_time'] * 1000)\n", " end_ms = round(segment['end_time'] * 1000)\n", " audio_segment = audio[start_ms:end_ms]\n", " segment_arr = np.array(audio_segment.get_array_of_samples())\n", " data.append(segment_arr)\n", " data.append(space_arr)\n", " new_start_ms = int(round(offset_frames / sample_rate * 1000))\n", " new_end_ms = new_start_ms + int(round(segment_arr.shape[0] / sample_rate * 1000))\n", " offset_frames += segment_arr.shape[0] + space_arr.shape[0]\n", " meta.append({\n", " \"start_ms\": new_start_ms,\n", " \"end_ms\": new_end_ms,\n", " \"sid\": segment['sid'],\n", " })\n", "new_audio = np.hstack(data)" ] }, { "cell_type": "code", "execution_count": 122, "id": "7a6b2c3e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " " ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "play_array(new_audio, sr=sample_rate)" ] }, { "cell_type": "code", "execution_count": 123, "id": "185c04a9", "metadata": {}, "outputs": [], "source": [ "wavfile.write(\"tmp/penn_station.wav\", sample_rate, new_audio)" ] }, { "cell_type": "code", "execution_count": 124, "id": "cbff1633", "metadata": {}, "outputs": [], "source": [ "with open(\"tmp/penn_station.json\", \"w\") as f:\n", " json.dump(meta, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "62933e03", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "94206d65", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b141a49a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c67eedf2", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }