{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "5fa84deb", "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 4, "id": "3dbc5484", "metadata": {}, "outputs": [], "source": [ "dfs = pd.read_html(\"https://www.shaip.com/offerings/speech-data-catalog/\")\n", "df = dfs[0]" ] }, { "cell_type": "code", "execution_count": 9, "id": "7f35a51c", "metadata": {}, "outputs": [], "source": [ "df.to_csv(\"shaip_catalog.csv\", index=False)" ] }, { "cell_type": "code", "execution_count": 11, "id": "25a2cc4b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(22, 15)\n" ] } ], "source": [ "_df = df[df[\"Language Dataset\"].str.contains(\"glish\")]\n", "print(_df.shape)" ] }, { "cell_type": "code", "execution_count": 13, "id": "5f84a53a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Language Dataset\n", "Chinese English 418\n", "English 10\n", "English Deep South 473\n", "English US 5\n", "Hinglish 424\n", "Hispanic English 367\n", "Indian English 337\n", "New York English 350\n", "New Zealand English 548\n", "Singapore English 465\n", "South African English 512\n", "Name: Total Audio Hours, dtype: int64" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "_df.groupby(\"Language Dataset\")[\"Total Audio Hours\"].sum()" ] }, { "cell_type": "code", "execution_count": 16, "id": "ef08b91d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Media Audio 36\n", "General Conversation 26\n", "Call-Center 25\n", "Scripted Monologue 15\n", "Call-center 1\n", "Media audio 1\n", "Name: Dataset Type, dtype: int64" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"Dataset Type\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 17, "id": "739c51a8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Unscripted, synthetic telephonic conversation between \"agent\" and \"customer\", Approx. Audio Duration (Range) 5-15 Minutes, 37\n", "Licensable Public domain audio/video files such as interviews, podcasts etc - 1 to 5 people. Approx. Audio Duration (Range) 15-60 minutes 37\n", "Single-utterance recordings, which tend to fall in the 5 to 30 second range 14\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, 8\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, Afrikaans spoken in Africa 1\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, Arabic from Gulf countries 1\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, Hebrew in Israel 1\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, Bahasa Indonesian 1\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, Malay in Malaysia 1\n", "Unscripted telephonic conversation between two people. Approx. Audio Duration (Range) - 15-60 minutes, An informal register used between friends 1\n", "Unscripted, synthetic telephonic conversation between \"agent\" and \"customer\", Approx. Audio Duration (Range) 5-15 Minutes, Northern (e.g.,Hanoi), Central, and Southern (e.g., Ho Chi Minh City). 1\n", "Single-utterance recordings, which tend to fall in the 5 to 30 second range, Danish from Denmark 1\n", "Name: Dataset description, dtype: int64" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"Dataset description\"].value_counts()" ] }, { "cell_type": "code", "execution_count": null, "id": "334b72a4", "metadata": {}, "outputs": [], "source": [ "Unscripted, synthetic telephonic conversation between \"agent\" and \"customer\". (2 channel)\n", "Licensable Public domain audio/video files such as interviews, podcasts etc - 1 to 5 people.\n", "Unscripted telephonic conversation between two people. (2 channel)\n", "Single-utterance recordings. (recorded via a Mobile App)" ] }, { "cell_type": "code", "execution_count": 18, "id": "328bb9fc", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(104, 15)" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shape" ] }, { "cell_type": "code", "execution_count": 27, "id": "29437b92", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "5.0 103\n", "Name: WER (%), dtype: int64" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"WER (%)\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 12, "id": "0cd962a4", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DetailsKeywordLanguage DatasetSample RateDataset TypeTotal Audio HoursTotal Speech HoursDataset descriptionAudio ChannelRecording PlatformWER (%)Audio FormatTranscription FormatUse CaseCTA
8NaNChinese EnglishChinese English8 kHzCall-Center169130Unscripted, synthetic telephonic conversation ...DualDesktop5.0.wav.jsonASR, Virtual Assistant, Chatbot, Conversationa...Contact
9NaNChinese EnglishChinese English16 kHzMedia Audio249236Licensable Public domain audio/video files suc...MonoDesktop5.0.wav.jsonASR, Virtual Assistant, Chatbot, Conversationa...Contact
12NaNEnglishEnglish16 kHzMedia Audio109Licensable Public domain audio/video files suc...MonoDesktop5.0.wav.jsonASR, Virtual Assistant, Chatbot, Conversationa...Contact
13NaNEnglish Deep SouthEnglish Deep South8 kHzCall-Center151149Unscripted, synthetic telephonic conversation ...DualDesktop5.0.wav.jsonASR, Virtual Assistant, Chatbot, Conversationa...Contact
14NaNEnglish Deep SouthEnglish Deep South8 kHzGeneral Conversation5656Unscripted telephonic conversation between two...DualDesktop5.0.wav.jsonASR, Virtual Assistant, Chatbot, Conversationa...Contact
\n", "
" ], "text/plain": [ " Details Keyword Language Dataset Sample Rate \\\n", "8 NaN Chinese English Chinese English 8 kHz \n", "9 NaN Chinese English Chinese English 16 kHz \n", "12 NaN English English 16 kHz \n", "13 NaN English Deep South English Deep South 8 kHz \n", "14 NaN English Deep South English Deep South 8 kHz \n", "\n", " Dataset Type Total Audio Hours Total Speech Hours \\\n", "8 Call-Center 169 130 \n", "9 Media Audio 249 236 \n", "12 Media Audio 10 9 \n", "13 Call-Center 151 149 \n", "14 General Conversation 56 56 \n", "\n", " Dataset description Audio Channel \\\n", "8 Unscripted, synthetic telephonic conversation ... Dual \n", "9 Licensable Public domain audio/video files suc... Mono \n", "12 Licensable Public domain audio/video files suc... Mono \n", "13 Unscripted, synthetic telephonic conversation ... Dual \n", "14 Unscripted telephonic conversation between two... Dual \n", "\n", " Recording Platform WER (%) Audio Format Transcription Format \\\n", "8 Desktop 5.0 .wav .json \n", "9 Desktop 5.0 .wav .json \n", "12 Desktop 5.0 .wav .json \n", "13 Desktop 5.0 .wav .json \n", "14 Desktop 5.0 .wav .json \n", "\n", " Use Case CTA \n", "8 ASR, Virtual Assistant, Chatbot, Conversationa... Contact \n", "9 ASR, Virtual Assistant, Chatbot, Conversationa... Contact \n", "12 ASR, Virtual Assistant, Chatbot, Conversationa... Contact \n", "13 ASR, Virtual Assistant, Chatbot, Conversationa... Contact \n", "14 ASR, Virtual Assistant, Chatbot, Conversationa... Contact " ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "_df.head()" ] }, { "cell_type": "code", "execution_count": null, "id": "eb57b4a7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "70935292", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3ab82919", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "540f909b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b92bf216", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9c9da4ad", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }