{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "fc68f037", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:20:04.280028Z", "start_time": "2023-10-24T17:20:04.202971Z" } }, "outputs": [], "source": [ "import os\n", "import glob\n", "import subprocess\n", "import tarfile\n", "import wget\n", "import copy\n", "from omegaconf import OmegaConf, open_dict" ] }, { "cell_type": "code", "execution_count": 2, "id": "3e60ab39", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:20:04.380015Z", "start_time": "2023-10-24T17:20:04.378138Z" } }, "outputs": [], "source": [ "data_dir = \"datasets/\"\n", "\n", "if not os.path.exists(data_dir):\n", " os.makedirs(data_dir, exist_ok=True)\n", "\n", "if not os.path.exists(\"scripts\"):\n", " os.makedirs(\"scripts\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "67809f28", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:20:09.839163Z", "start_time": "2023-10-24T17:20:04.542814Z" } }, "outputs": [], "source": [ "import nemo\n", "import nemo.collections.asr as nemo_asr\n", "from nemo.collections.asr.metrics.wer import word_error_rate\n", "from nemo.utils import logging, exp_manager" ] }, { "cell_type": "code", "execution_count": 10, "id": "3f961c59", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:25:11.069818Z", "start_time": "2023-10-24T17:25:09.954576Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2023-10-24 17:25:09-- https://github.com/NVIDIA/NeMo/blob/main/scripts/dataset_processing/get_commonvoice_data.py\n", "Resolving github.com (github.com)... 20.27.177.113\n", "Connecting to github.com (github.com)|20.27.177.113|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 50453 (49K) [text/plain]\n", "Saving to: ‘scripts/get_commonvoice_data.py’\n", "\n", "get_commonvoice_dat 100%[===================>] 49.27K --.-KB/s in 0.1s \n", "\n", "2023-10-24 17:25:10 (518 KB/s) - ‘scripts/get_commonvoice_data.py’ saved [50453/50453]\n", "\n" ] } ], "source": [ "if not os.path.exists(\"scripts/get_commonvoice_data.py\"):\n", " !wget -P scripts/ https://github.com/NVIDIA/NeMo/blob/main/scripts/dataset_processing/get_commonvoice_data.py" ] }, { "cell_type": "code", "execution_count": 11, "id": "d4a76d69", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:25:13.073112Z", "start_time": "2023-10-24T17:25:13.070691Z" } }, "outputs": [], "source": [ "VERSION = \"cv-corpus-6.1-2020-12-11\"\n", "LANGUAGE = \"ja\"" ] }, { "cell_type": "code", "execution_count": 12, "id": "1997ad2e", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:25:13.276519Z", "start_time": "2023-10-24T17:25:13.274644Z" } }, "outputs": [], "source": [ "tokenizer_dir = os.path.join('tokenizers', LANGUAGE)\n", "manifest_dir = os.path.join('manifests', LANGUAGE)" ] }, { "cell_type": "code", "execution_count": 13, "id": "250f0276", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:25:13.584569Z", "start_time": "2023-10-24T17:25:13.454464Z" } }, "outputs": [], "source": [ "!mkdir -p datasets" ] }, { "cell_type": "code", "execution_count": 16, "id": "20f9cd93", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:27:56.817676Z", "start_time": "2023-10-24T17:26:37.085082Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2023-10-24 17:26:37-- https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/cv-corpus-6.1-2020-12-11/ja.tar.gz\n", "Resolving voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com (voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com)... 52.92.208.161, 52.92.179.25, 52.92.162.57, ...\n", "Connecting to voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com (voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com)|52.92.208.161|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 152879796 (146M) [application/octet-stream]\n", "Saving to: ‘datasets/ja/ja.tar.gz’\n", "\n", "datasets/ja/ja.tar. 100%[===================>] 145.80M 15.5MB/s in 9.9s \n", "\n", "2023-10-24 17:26:48 (14.7 MB/s) - ‘datasets/ja/ja.tar.gz’ saved [152879796/152879796]\n", "\n", "100%|████████████████████████████████████████| 721/721 [00:02<00:00, 242.55it/s]\n", "100%|█████████████████████████████████████| 721/721 [00:00<00:00, 128630.08it/s]\n", "100%|████████████████████████████████████████| 585/585 [00:02<00:00, 237.13it/s]\n", "100%|█████████████████████████████████████| 585/585 [00:00<00:00, 123256.53it/s]\n", "100%|████████████████████████████████████████| 631/631 [00:02<00:00, 240.51it/s]\n", "100%|█████████████████████████████████████| 631/631 [00:00<00:00, 128669.64it/s]\n" ] } ], "source": [ "!python scripts/get_commonvoice_data.py \\\n", " --data_root \"datasets/$LANGUAGE/\" \\\n", " --manifest_dir=$manifest_dir \\\n", " --sample_rate=16000 \\\n", " --n_channels=1 \\\n", " --version=$VERSION \\\n", " --language=$LANGUAGE \\\n", " --files_to_process 'train.tsv' 'dev.tsv' 'test.tsv'" ] }, { "cell_type": "code", "execution_count": 17, "id": "9e43c8da", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:27:56.822872Z", "start_time": "2023-10-24T17:27:56.820138Z" } }, "outputs": [], "source": [ "train_manifest = f\"{manifest_dir}/commonvoice_train_manifest.json\"\n", "dev_manifest = f\"{manifest_dir}/commonvoice_dev_manifest.json\"\n", "test_manifest = f\"{manifest_dir}/commonvoice_test_manifest.json\"" ] }, { "cell_type": "code", "execution_count": 18, "id": "08fdf1e5", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:28:29.212850Z", "start_time": "2023-10-24T17:28:29.204721Z" } }, "outputs": [], "source": [ "# Manifest Utils\n", "from tqdm.auto import tqdm\n", "from nemo.collections.asr.parts.utils.manifest_utils import read_manifest, write_manifest\n", "import json\n", "\n", "\n", "def write_processed_manifest(data, original_path):\n", " original_manifest_name = os.path.basename(original_path)\n", " new_manifest_name = original_manifest_name.replace(\".json\", \"_processed.json\")\n", "\n", " manifest_dir = os.path.split(original_path)[0]\n", " filepath = os.path.join(manifest_dir, new_manifest_name)\n", " write_manifest(filepath, data)\n", " print(f\"Finished writing manifest: {filepath}\")\n", " return filepath" ] }, { "cell_type": "code", "execution_count": 19, "id": "05807009", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:28:35.067622Z", "start_time": "2023-10-24T17:28:35.058542Z" } }, "outputs": [], "source": [ "train_manifest_data = read_manifest(train_manifest)\n", "dev_manifest_data = read_manifest(dev_manifest)\n", "test_manifest_data = read_manifest(test_manifest)" ] }, { "cell_type": "code", "execution_count": 20, "id": "d3699205", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:28:39.029833Z", "start_time": "2023-10-24T17:28:39.027552Z" } }, "outputs": [], "source": [ "train_text = [data['text'] for data in train_manifest_data]\n", "dev_text = [data['text'] for data in dev_manifest_data]\n", "test_text = [data['text'] for data in test_manifest_data]" ] }, { "cell_type": "code", "execution_count": 21, "id": "bdb3aa24", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:28:46.690890Z", "start_time": "2023-10-24T17:28:46.688347Z" } }, "outputs": [], "source": [ "from collections import defaultdict\n", "\n", "def get_charset(manifest_data):\n", " charset = defaultdict(int)\n", " for row in tqdm(manifest_data, desc=\"Computing character set\"):\n", " text = row['text']\n", " for character in text:\n", " charset[character] += 1\n", " return charset" ] }, { "cell_type": "code", "execution_count": 22, "id": "3a1821a5", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:28:52.623033Z", "start_time": "2023-10-24T17:28:52.598785Z" } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4eb3c524943a49babeb7c46691b96e46", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Computing character set: 0%| | 0/721 [00:00" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "plt.bar(x=TOKEN_COUNT_X, height=NUM_TOKENS_Y)\n", "plt.title(\"Occurrences of unique tokens in train+dev set\")\n", "plt.xlabel(\"# of occurrences\")\n", "plt.ylabel(\"# of tokens\")\n", "plt.xlim(0, MAX_COUNT);" ] }, { "cell_type": "code", "execution_count": 30, "id": "451d6f3b", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:32:15.849045Z", "start_time": "2023-10-24T17:32:15.845977Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of tokens with <= 5 occurrences : 1040\n" ] } ], "source": [ "UNCOMMON_TOKENS_COUNT = 5\n", "\n", "chars_with_infrequent_occurrence = set()\n", "for count in range(1, UNCOMMON_TOKENS_COUNT + 1):\n", " if count in train_counts:\n", " token_list = train_counts[count]\n", " chars_with_infrequent_occurrence.update(set(token_list))\n", "\n", "print(f\"Number of tokens with <= {UNCOMMON_TOKENS_COUNT} occurrences : {len(chars_with_infrequent_occurrence)}\")" ] }, { "cell_type": "code", "execution_count": 31, "id": "a19aefb6", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:32:23.410206Z", "start_time": "2023-10-24T17:32:23.407842Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Original train+dev+test vocab size : 1431\n", "New train vocab size : 1254\n" ] } ], "source": [ "all_tokens = set.union(train_dev_set, test_set)\n", "print(f\"Original train+dev+test vocab size : {len(all_tokens)}\")\n", "\n", "extra_kanji = set(test_oov)\n", "train_token_set = all_tokens - extra_kanji\n", "print(f\"New train vocab size : {len(train_token_set)}\")" ] }, { "cell_type": "code", "execution_count": 32, "id": "b4f417ff", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:33:13.625348Z", "start_time": "2023-10-24T17:33:13.622657Z" } }, "outputs": [], "source": [ "import unicodedata\n", "def process_dakuten(text):\n", " normalized_text = unicodedata.normalize('NFD', text)\n", " normalized_text = normalized_text.replace(\"\\u3099\", \"\").replace(\"\\u309A\", \"\")\n", " return normalized_text" ] }, { "cell_type": "code", "execution_count": 34, "id": "0027d318", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:33:37.825896Z", "start_time": "2023-10-24T17:33:37.822623Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "After dakuten normalization, number of train tokens : 1209\n" ] } ], "source": [ "PERFORM_DAKUTEN_NORMALIZATION = True\n", "if PERFORM_DAKUTEN_NORMALIZATION:\n", " normalized_train_token_set = set()\n", " for token in train_token_set:\n", " normalized_token = process_dakuten(str(token))\n", " normalized_train_token_set.update(normalized_token)\n", " \n", " print(f\"After dakuten normalization, number of train tokens : {len(normalized_train_token_set)}\")\n", "else:\n", " normalized_train_token_set = train_token_set" ] }, { "cell_type": "code", "execution_count": 35, "id": "86c8d2b2", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:33:48.902309Z", "start_time": "2023-10-24T17:33:48.899114Z" } }, "outputs": [], "source": [ "# Preprocessing steps\n", "import re\n", "import unicodedata\n", "\n", "chars_to_ignore_regex = '[\\,\\?\\.\\!\\-\\;\\:\\\"\\“\\%\\‘\\”\\�\\…\\{\\}\\【\\】\\・\\。\\『\\』\\、\\ー\\〜]' # remove special character tokens\n", "kanji_removal_regex = '[' + \"\".join([f\"\\{token}\" for token in extra_kanji]) + ']' # remove test set kanji\n", "\n", "\n", "def remove_special_characters(data):\n", " data[\"text\"] = re.sub(chars_to_ignore_regex, '', data[\"text\"]).lower().strip()\n", " return data\n", "\n", "def remove_extra_kanji(data):\n", " data[\"text\"] = re.sub(kanji_removal_regex, '', data[\"text\"])\n", " return data\n", "\n", "def remove_dakuten(data):\n", " # perform dakuten normalization (if it was requested)\n", " if PERFORM_DAKUTEN_NORMALIZATION:\n", " text = data['text']\n", " data['text'] = process_dakuten(text)\n", " return data" ] }, { "cell_type": "code", "execution_count": 36, "id": "412ba8ad", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:33:55.580932Z", "start_time": "2023-10-24T17:33:55.578762Z" } }, "outputs": [], "source": [ "# Processing pipeline\n", "def apply_preprocessors(manifest, preprocessors):\n", " for processor in preprocessors:\n", " for idx in tqdm(range(len(manifest)), desc=f\"Applying {processor.__name__}\"):\n", " manifest[idx] = processor(manifest[idx])\n", "\n", " print(\"Finished processing manifest !\")\n", " return manifest" ] }, { "cell_type": "code", "execution_count": 37, "id": "318ca8e2", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:33:59.716386Z", "start_time": "2023-10-24T17:33:59.714474Z" } }, "outputs": [], "source": [ "# List of pre-processing functions\n", "PREPROCESSORS = [\n", " remove_special_characters,\n", " remove_extra_kanji,\n", " remove_dakuten,\n", "]" ] }, { "cell_type": "code", "execution_count": 38, "id": "07935544", "metadata": { "ExecuteTime": { "end_time": "2023-10-24T17:34:05.928394Z", "start_time": "2023-10-24T17:34:05.811907Z" } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4ac3535a8a034e10b2dd1148e90c069f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Applying remove_special_characters: 0%| | 0/721 [00:00