{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "e7eb03ea", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Populating the interactive namespace from numpy and matplotlib\n" ] } ], "source": [ "%pylab inline" ] }, { "cell_type": "code", "execution_count": 2, "id": "fa1a598b", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "a9594109", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "import pandas as pd\n", "import json\n", "\n", "from suno_utils.utils.notebook import _" ] }, { "cell_type": "code", "execution_count": null, "id": "18e74570", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "5574012e", "metadata": {}, "source": [ "## get transcripts" ] }, { "cell_type": "code", "execution_count": null, "id": "04d2b3ed", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import get_auth_string\n", "from suno_utils.web.rev import get_transcript, wait_till_done\n", "\n", "client_api_key = \"_CbCcpYaSu4DF8NuQJdWniWG2Ls\"\n", "user_api_key = \"hYhwAFC3w39VN23lwpzn26Jy1CA=\"\n", "\n", "AUTH_STR = get_auth_string(client_api_key, user_api_key)\n", "transcript_ids = wait_till_done(AUTH_STR, \"TC0299517215\")\n", "transcripts = [get_transcript(AUTH_STR, transcript_id) for transcript_id in transcript_ids]" ] }, { "cell_type": "code", "execution_count": null, "id": "3adf47a8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9ecd8f80", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "3d644332", "metadata": {}, "source": [ "## Test Transcript parser" ] }, { "cell_type": "code", "execution_count": null, "id": "2022dc97", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.rev import parse_transcript\n", "from suno_utils.utils.alignment import format_tokens_as_turns\n", "\n", "with open(\"tmp/bla.pkl\", \"rb\") as f:\n", " transcripts = pickle.load(f)" ] }, { "cell_type": "code", "execution_count": 5, "id": "9aaa52b6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "87.3% tokens aligned\n" ] } ], "source": [ "speaker_names, align_meta = parse_transcript(transcripts[9])\n", "print(\"{}% tokens aligned\".format(round(np.mean([t[\"success\"] for t in align_meta if t[\"type\"] == \"text\"]) * 100, 1)))" ] }, { "cell_type": "code", "execution_count": 88, "id": "103ee71c", "metadata": { "scrolled": false }, "outputs": [], "source": [ "# for s, t in format_tokens_as_turns(align_meta):\n", "# print(f\"Speaker {s}:\")\n", "# print(t)\n", "# print()" ] }, { "cell_type": "code", "execution_count": null, "id": "540622fd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ec5165fc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 6, "id": "9cb590c4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "aussie_100\t\t find_cc_podcasts.ipynb old\t\t\t tmp\r\n", "bulk_download_rss.ipynb find_india.ipynb\t organize_metadata.ipynb\r\n" ] } ], "source": [ "!ls /home/georg/notebooks/datasets/podcasts/" ] }, { "cell_type": "code", "execution_count": null, "id": "fa08336e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3fd494f4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c222b398", "metadata": {}, "source": [ "## Make slices" ] }, { "cell_type": "code", "execution_count": 5, "id": "4fbc805a", "metadata": {}, "outputs": [], "source": [ "import os\n", "import pickle\n", "import shutil\n", "\n", "from suno_utils.utils.conversion import Audio\n", "from suno_utils.utils.slicer import get_annotated_segments\n", "from suno_utils.web.rev import parse_transcript\n", "\n", "DATA_DIR = \"/home/georg/notebooks/datasets/podcasts/aussie_100/\"" ] }, { "cell_type": "code", "execution_count": 106, "id": "6a27a54f", "metadata": {}, "outputs": [], "source": [ "# shutil.rmtree(DATA_DIR, ignore_errors=True)\n", "# os.makedirs(DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 7, "id": "85cc3693", "metadata": {}, "outputs": [], "source": [ "episode_metas = pd.read_csv(\"old/10_ep_aussie_100.csv\").to_dict(orient=\"records\")\n", "with open(\"tmp/bla.pkl\", \"rb\") as f:\n", " transcripts = pickle.load(f)" ] }, { "cell_type": "code", "execution_count": 8, "id": "d3a70ff9", "metadata": {}, "outputs": [], "source": [ "def _get_podcast_segments(audio, transcript):\n", " speaker_names, align_meta = parse_transcript(transcript)\n", " segments = get_annotated_segments(audio, align_meta)\n", " return segments" ] }, { "cell_type": "code", "execution_count": 33, "id": "05287688", "metadata": {}, "outputs": [], "source": [ "# test slice making\n", "# episode_meta, transcript = episode_metas[3], transcripts[3]\n", "\n", "# audio = Audio.from_file(\"old/\" + episode_meta[\"wav_filepath\"])\n", "# slice_meta = _get_podcast_slices(audio, transcript)\n", "\n", "# slice_cov_frac = np.sum([to_s - from_s for _, (from_s, to_s), _ in slice_meta]) / audio.duration_s\n", "# print(\"{}% slice coverage\".format(round(slice_cov_frac * 100, 1)))\n", "# # 97.0% slice coverage" ] }, { "cell_type": "code", "execution_count": 173, "id": "50569c74", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "10it [00:04, 2.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "91.3% avg segment coverage\n", "133 total segments\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "cov_l = []\n", "segment_metas = []\n", "for episode_meta, transcript in tqdm.tqdm(zip(episode_metas, transcripts)):\n", " audio = Audio.from_file(\"old/\" + episode_meta[\"wav_filepath\"])\n", " segments = _get_podcast_segments(audio, transcript)\n", " segment_cov_frac = np.sum([to_s - from_s for _, (from_s, to_s), _ in segments]) / audio.duration_s\n", " cov_l.append(segment_cov_frac)\n", " # skip everything pre-1min\n", " segments = [e for e in segments if e[1][0] >= 60]\n", " # write segments and make metadata\n", " for segment_audio, (start_s, end_s), tokens in segments:\n", " start_ms = int(start_s * 1_000)\n", " end_ms = int(end_s * 1_000)\n", " segment_id = f\"{episode_meta['file_id']}__{start_ms}__{end_ms}\"\n", " segment_filepath = f\"{DATA_DIR}segments/{episode_meta['file_id']}/{start_ms}__{end_ms}.wav\"\n", " os.makedirs(os.path.dirname(segment_filepath), exist_ok=True)\n", " segment_audio.write_wav(segment_filepath)\n", " segment_metas.append({\n", " \"source_id\": episode_meta[\"file_id\"],\n", " \"source_offset_s\": start_s,\n", " \"id\": segment_id,\n", " \"uri\": segment_filepath,\n", " \"duration_s\": end_s - start_s,\n", " \"transcript\": {\n", " \"tokens\": tokens,\n", " },\n", " })\n", "print(\"{}% avg segment coverage\".format(round(np.mean(cov_l) * 100, 1)))\n", "print(len(segment_metas), \"total segments\")\n", "# 91.3% avg slice coverage\n", "# 133 total slices" ] }, { "cell_type": "code", "execution_count": 20, "id": "7dd865c4", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"segment_meta.json\", \"w\") as f:\n", " json.dump(segment_metas, f)" ] }, { "cell_type": "code", "execution_count": 22, "id": "bc93fe1d", "metadata": { "scrolled": false }, "outputs": [], "source": [ "# from suno_utils.utils.alignment import format_tokens_as_turns\n", "# import random\n", "\n", "# idx_list = list(range(len(segment_metas)))\n", "# random.shuffle(idx_list)\n", "\n", "# for idx in idx_list[:5]:\n", "# segment_meta = segment_metas[idx]\n", "# print(\"-\" * 10)\n", "# print(f\"({segment_meta['id']})\")\n", "# Audio.from_file(segment_meta[\"filepath\"]).play()\n", "# for s, t in format_tokens_as_turns(segment_meta[\"transcript\"][\"tokens\"]):\n", "# print()\n", "# print(f\"Speaker {s}:\")\n", "# print(f\"{t}\")\n", "# print()" ] }, { "cell_type": "code", "execution_count": null, "id": "258c06f6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3f15a38e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "18ff5264", "metadata": {}, "source": [ "## Normalization" ] }, { "cell_type": "code", "execution_count": 27, "id": "932d26ad", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.notebook import transcribe, Audio\n", "\n", "DATA_DIR = \"/home/georg/notebooks/datasets/podcasts/aussie_100/\"\n", "\n", "with open(DATA_DIR + \"segment_meta.json\") as f:\n", " segment_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 26, "id": "a4cf166c", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Transcribing: 0%| | 0/17 [00:00 0:\n", " turns.append((cur_speaker, \" \".join(tmp)))\n", " tmp = []\n", " tmp.append(token[\"value\"])\n", " cur_speaker = token[\"speaker_id\"]\n", " if len(tmp) > 0:\n", " turns.append((cur_speaker, \" \".join(tmp)))\n", " return turns" ] }, { "cell_type": "code", "execution_count": 32, "id": "8ea64a52", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.text_normalizer import normalize, normalize_mp\n", "from suno_utils.utils.aligner import format_tokens_as_turns\n", "\n", "transcript_turns = []\n", "for segment_meta in segment_metas:\n", " transcript_turns.append([\n", " t for _, t in format_tokens_as_turns(segment_meta[\"transcript\"][\"tokens\"], tag_repr_map={})\n", " ])\n", " \n", "_, transcript_turns_norm = normalize_mp(transcript_turns, asr_predictions, n_cores=10)" ] }, { "cell_type": "code", "execution_count": 91, "id": "c0a6aaff", "metadata": {}, "outputs": [], "source": [ "for segment_meta, turns_norm, asr_pred in zip(segment_metas, transcript_turns_norm, asr_predictions):\n", " segment_meta[\"transcript\"][\"text_normalized\"] = \" \".join(turns_norm)\n", " segment_meta[\"transcript\"][\"asr_prediction\"] = asr_pred" ] }, { "cell_type": "code", "execution_count": null, "id": "c4c90bd9", "metadata": {}, "outputs": [], "source": [ "# TODO: rank prior on norms to choose if nemo facepalms and cer all high\n", "# normalize(transcript_turns[8], asr_predictions[8])" ] }, { "cell_type": "code", "execution_count": 93, "id": "450fcb45", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"segment_meta_post_norm.json\", \"w\") as f:\n", " json.dump(segment_metas, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "ae66c0e7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c4eacfe1", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0b63ef6a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "e090b7ce", "metadata": {}, "source": [ "## Timestamps (optional)" ] }, { "cell_type": "code", "execution_count": 10, "id": "7144455d", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.notebook import transcribe, Audio\n", "\n", "DATA_DIR = \"/home/georg/notebooks/datasets/podcasts/aussie_100/\"\n", "\n", "with open(DATA_DIR + \"segment_meta_post_norm.json\") as f:\n", " segment_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 13, "id": "e4c6a6f7", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.mfa import align_tokens, align_text\n", "\n", "conda_sh_filepath = \"/home/georg/anaconda3/etc/profile.d/conda.sh\"\n", "env_name = \"mfa\"\n", "\n", "audio_filepaths = [e[\"uri\"] for e in segment_metas]" ] }, { "cell_type": "code", "execution_count": 36, "id": "26afeda9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0/133 failed.\n" ] } ], "source": [ "transcripts_tokens = [e[\"transcript\"][\"tokens\"] for e in segment_metas]\n", "word_aligments = align_tokens(audio_filepaths, transcripts_tokens, conda_sh_filepath, env_name, num_cores=5)\n", "n_failed = len(transcripts_tokens) - len([_ for e in word_aligments if e is not None])\n", "print(\"{}/{} failed.\".format(n_failed, len(transcripts_tokens)))" ] }, { "cell_type": "code", "execution_count": 38, "id": "55fd5fbc", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0/133 failed.\n" ] } ], "source": [ "transcripts_norm = [e[\"transcript\"][\"text_normalized\"] for e in segment_metas]\n", "word_aligments_norm = align_text(audio_filepaths, transcripts_norm, conda_sh_filepath, env_name, num_cores=5)\n", "n_failed = len(transcripts_norm) - len([_ for e in word_aligments_norm if e is not None])\n", "print(\"{}/{} failed.\".format(n_failed, len(transcripts_norm)))" ] }, { "cell_type": "code", "execution_count": 62, "id": "c978d868", "metadata": {}, "outputs": [], "source": [ "for segment_meta, word_aligment, word_aligment_norm in zip(segment_metas, word_aligments, word_aligments_norm):\n", " segment_meta[\"transcript\"][\"alignment\"] = word_aligment\n", " segment_meta[\"transcript\"][\"alignment\"] = word_aligment_norm" ] }, { "cell_type": "code", "execution_count": 44, "id": "ab1a05cc", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"segment_meta_post_align.json\", \"w\") as f:\n", " json.dump(segment_metas, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "7496ca48", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "76d3071a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c2a8c069", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "6db01585", "metadata": {}, "source": [ "## deverbalizer (optional)" ] }, { "cell_type": "code", "execution_count": 15, "id": "e89f9e21", "metadata": {}, "outputs": [], "source": [ "# from suno_utils.utils.text_denormalizer import denormalize\n", "\n", "# # denormalize(\" I -- I -- I -- I like coffee, coffee\")\n", "# denormalize(\"yesterday I -- anyways, [laughter], I like coffee, coffee\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a7b2b8dc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6991ecfd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "632ebdea", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "4f94a394", "metadata": {}, "source": [ "## Filter (asr based etc)" ] }, { "cell_type": "code", "execution_count": 102, "id": "e15eba07", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"segment_meta_post_norm.json\") as f:\n", " segment_metas = json.load(f)" ] }, { "cell_type": "code", "execution_count": 103, "id": "45a937ae", "metadata": {}, "outputs": [], "source": [ "from nemo.collections.asr.metrics.wer import word_error_rate\n", "\n", "def _get_cer(a, b):\n", " if a == b:\n", " return 0\n", " cer = word_error_rate([a], [b], use_cer=True)\n", " return cer" ] }, { "cell_type": "code", "execution_count": 105, "id": "a1df4e61", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.aligner import format_tokens_as_turns\n", "\n", "metric_df = pd.DataFrame({\n", " \"transcript\": [\" \".join([t for _, t in format_tokens_as_turns(e[\"transcript\"][\"tokens\"])]) for e in segment_metas],\n", " \"transcript_norm\": [e[\"transcript\"][\"text_normalized\"] for e in segment_metas],\n", " \"asr_pred\": [e[\"transcript\"][\"asr_prediction\"] for e in segment_metas],\n", "})\n", "metric_df[\"cer\"] = [\n", " _get_cer(a, b) for a, b in zip(metric_df[\"transcript_norm\"].values, metric_df[\"asr_pred\"].values)\n", "]" ] }, { "cell_type": "code", "execution_count": 107, "id": "f17fac80", "metadata": {}, "outputs": [], "source": [ "# for idx, row in metric_df[metric_df[\"transcript\"].str.contains(r\"[0-9]\")].sort_values(\"cer\", ascending=False).head(20).iterrows():\n", "# print(\"idx: {}, cer: {}%\".format(idx, round(row[\"cer\"] * 100, 1)))\n", "# print(\" \" + row[\"transcript\"])\n", "# print(\" \" + row[\"transcript_norm\"])\n", "# print(\" \" + row[\"asr_pred\"])\n", "# print()" ] }, { "cell_type": "code", "execution_count": null, "id": "1d6a879c", "metadata": {}, "outputs": [], "source": [ "# TODO: Mm-hmm vs mhm\n", "# TODO: contractions don't become expanded\n", "\n", "# TODO: improvements for normalizer\n", "# can we add custom rules?\n", "# can we keep track of replacement indices (worst case difflib)" ] }, { "cell_type": "code", "execution_count": 169, "id": "add0be12", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"segment_meta_post_filter.json\", \"w\") as f:\n", " json.dump(segment_metas, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "ea37d95c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "70b83be3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "75abd2d5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "bc900f73", "metadata": {}, "source": [ "## corpus distribution" ] }, { "cell_type": "code", "execution_count": 177, "id": "e1a3ca09", "metadata": {}, "outputs": [], "source": [ "with open(DATA_DIR + \"segment_meta_post_filter.json\") as f:\n", " segment_metas = json.load(f)\n", " \n", "DIST_DIR = \"/home/georg/notebooks/datasets/podcasts/aussie_100/dist/\"" ] }, { "cell_type": "code", "execution_count": 134, "id": "7547d9c4", "metadata": {}, "outputs": [], "source": [ "# !mkdir /home/georg/notebooks/datasets/podcasts/aussie_100/dist\n", "# !mkdir /home/georg/notebooks/datasets/podcasts/aussie_100/dist/aussie_100\n", "# !mkdir /home/georg/notebooks/datasets/podcasts/aussie_100/dist/aussie_100/segments" ] }, { "cell_type": "code", "execution_count": 178, "id": "980f633b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "100 slices\n" ] } ], "source": [ "import shutil\n", "import copy\n", "import hashlib\n", "\n", "def _get_hash(s, n=None):\n", " uuid = hashlib.sha256(str(s).encode(\"utf-8\")).hexdigest()\n", " if n is not None:\n", " uuid = uuid[:n]\n", " return uuid\n", "\n", "random.seed(6006)\n", "idx_list = list(range(len(segment_metas)))\n", "random.shuffle(idx_list)\n", "\n", "out_segments_meta = []\n", "for idx in idx_list:\n", " segment_meta = segment_metas[idx]\n", " segment_id = _get_hash(segment_meta[\"id\"], n=16)\n", " \n", " from_filepath = segment_meta[\"uri\"]\n", " segment_filepath = \"segments/{}.wav\".format(segment_id)\n", " to_filepath = DIST_DIR + \"aussie_100/\" + segment_filepath\n", " shutil.copy(from_filepath, to_filepath)\n", " \n", " out_tokens = []\n", " for token in segment_meta[\"transcript\"][\"tokens\"]:\n", " out_token = {\n", " \"type\": token[\"type\"],\n", " \"value\": token[\"value\"],\n", " }\n", " if \"speaker_id\" in token:\n", " out_token[\"speaker_id\"] = _get_hash(segment_meta[\"source_id\"] + str(token[\"speaker_id\"]), n=16)\n", " out_tokens.append(out_token)\n", " \n", " out_segments_meta.append({\n", " \"source_id\": _get_hash(segment_meta[\"source_id\"], n=16),\n", " \"uid\": segment_id,\n", " \"filepath\": segment_filepath,\n", " \"duration_s\": round(segment_meta[\"duration_s\"], 2),\n", " \"transcript\": {\n", " \"text_normalized\": segment_meta[\"transcript\"][\"text_normalized\"],\n", " \"tokens\": out_tokens,\n", " }\n", " })\n", " if len(out_segments_meta) == 100:\n", " break\n", " \n", "print(len(out_segments_meta), \"segments\")" ] }, { "cell_type": "code", "execution_count": 168, "id": "77dc7378", "metadata": {}, "outputs": [], "source": [ "# print(json.dumps(out_segments_meta[1], indent=2))" ] }, { "cell_type": "code", "execution_count": 179, "id": "f3cae4a4", "metadata": {}, "outputs": [], "source": [ "# with open(DIST_DIR + \"aussie_100/meta.jsonl\", \"w\") as f:\n", "# for m in out_segments_meta:\n", "# f.write(json.dumps(m) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": 180, "id": "3dd23aa1", "metadata": {}, "outputs": [], "source": [ "# !cd /home/georg/notebooks/datasets/podcasts/aussie_100/dist && tar -czf aussie_100.tar.gz aussie_100" ] }, { "cell_type": "code", "execution_count": null, "id": "1d2e1868", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e90e8d0f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "10a4c391", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "612e707f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1d907dbd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1733b3e3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "1c313f0c", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 102, "id": "fe75bce3", "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAx8AAADQCAYAAABm8txZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAo6klEQVR4nO3deXxU9b3/8fc3mUkmG1khbMHgyiIoEBWNVq0WEResimhrxav32lbtta1a0Wpdqvfn75b2erWirV6v6M+lgLiguAt1AUSgyqpVWmQTCCEJhOzJ9/fHmUlmORMSIGdIfD0fDx6Z+Z7vOed7znc582HO94yx1goAAAAAulpSogsAAAAA4NuB4AMAAACAJwg+AAAAAHiC4AMAAACAJwg+AAAAAHiC4AMAAACAJ3ydyVxQUGCLi4u7qCgAAAAAurtly5btsNb2dlvWqeCjuLhYS5cuPTClAgAAANDjGGO+jreM264AAAAAeILgAwAAAIAnOnXbVaJV7GnQe59vT3QxgG7nxMPy1T8nLdHF0MJ1O/RNZV2iiwEA32opviSdd0z/RBej1Ut/26zmFpvoYhzUjh+cp6K89EQX44DoVsHH5spa3Tjrs0QXA+h2Hrui5KAIPv73o/V6e822RBcDAL7VctP9B1XwccsLK1Tf1JLoYhzUHrpsVI8JPrjtCgAAAIAnCD4AAAAAeILgAwAAAIAnutWcj+w0vy4cNSDRxQC6nX7ZgUQXQZI09tB8ZaV2q2EHAHqc9NTkRBchwsRj+6upmQnn7RmQm/h5mweKsbbjlV1SUmL5kUEAAAAA8RhjlllrS9yWcdsVAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwBMEHAAAAAE8QfAAAAADwhC/RBQAAAAC6SmNjozZt2qS6urpEF6XHCQQCGjhwoPx+f4fXIfgAAABAj7Vp0yZlZWWpuLhYxphEF6fHsNaqvLxcmzZt0uDBgzu8HrddAQAAoMeqq6tTfn4+gccBZoxRfn5+p79RIvgAAABAj0bg0TX25bwSfAAAAADwBMEHAAAA4LHTTjtNS5culSRNmDBBlZWViS2QR5hwDgAAACTQvHnzEl0EzxB8AAAAoMcrnvpal+9j/f3nuKbv2bNHl1xyiTZt2qTm5mbdcccdkWUrLtbSpUtVUFCgp556StOmTZMxRiNHjtTTTz+tsrIy/eQnP9GGDRskSQ888IBKS0v117/+VTfccIMkZ/7F+++/r6ysrK49yP1E8AEAAAB0oTfeeEP9+/fXa685AVBVVZUeeeSRmHyrV6/Wvffeq4ULF6qgoEA7d+6UJN1www36xS9+oZNPPlkbNmzQWWedpbVr12ratGl6+OGHVVpaqurqagUCAU+Pa18QfAAAAABdaMSIEbrxxht1yy236Nxzz9Upp5zimu+9997TpEmTVFBQIEnKy8uTJL3zzjtas2ZNa75du3apurpapaWl+uUvf6kf/vCHuvDCCzVw4MCuP5j9RPABAAAAdKEjjzxSy5cv17x583T77bfrjDPO6NT6LS0tWrx4ccw3G1OnTtU555yjefPmqbS0VG+++aaGDBlyIIt+wPG0KwAAAKALbdmyRenp6br88st18803a/ny5a75vvvd72rWrFkqLy+XpNbbrsaNG6eHHnqoNd+nn34qSVq3bp1GjBihW265Rccdd5w+//zzrj2QA4BvPgAAANDjxZsM7oWVK1fq5ptvVlJSkvx+vx555BHddNNNMfmGDx+uX//61zr11FOVnJysUaNG6cknn9SDDz6o6667TiNHjlRTU5O+853v6NFHH9UDDzyg+fPnKykpScOHD9fZZ5+dgKPrHGOt7XDmkpISG3oeMQAAAHCwW7t2rYYOHZroYvRYbufXGLPMWlvilp/brgAAAAB4guADAAAAgCcIPgAAAAB4guADAAAAgCcIPgAAAAB4guADAAAAgCcIPgAAAICD1F133aVp06YluhgHTPf6kcGanVJZnF9u3LZayi2WUjKkqk1S5QYnfdCJkjHO8tQsKWeQk169XSr/Suo/SvKntW2nepuUWRi5bV9AGjA6dn91VZFpvYdI6Xlt75sapM0uv4uSd5i0c11kWl2VU+7wbSb7paMmSAVHSBVfS7s2O+mbPpGaG6XsgW3Hk+SXNn4sNdWFlecoqfgUKS0ntgwNNdI3n0qbl0lN9dKWv0m2RRowRuo7UkrNbPsrSWVfSNtWOce96RNp0zKpcY+zLDVbSkqWTDCWrdnRth9/hjTsfKesfYZJWX2dOqzZKWUXSTlFUuVGqeKfUlJYc6xYL9XvltJypX8scJZLTp6s/k658g516rvgSGdZn6HSmlekf74v7d4Se8ySVHCUdPSFUr9jpNzBUu1O55xvW+2cv+2fS72PdNpEY63TFkZd3rZ+Y21kewnJLJTyD3Neb17eVg9bVzrHUThcCmS3lTMtN3YbFeulNS87bbdivZNWv1safKrUq5/Tvqo2Oee5V/+29Xr1d9p+2RfSjr87f0PScp39RbfNfVW5wTlX21a3pfkC0uBTnHMaz44vnb61YXFbWm5x23H0HyXt2SFVbYxdd/dWKdBL8qc776u3SWvnOu0p3KATpUFjnbYYsm2NVFfpXqbwOgvV05qXnXFBknIOkWrKnTyDTpJSgvsPtRdJqq1w2nPh0dKRZzlpJlkadELbfqo2S5Vfx+7/kJNi03Z909bWQ9LypD5D3I8h2va1TplCtvzNabPJfmngcU5a7mCnLX7zaez69budcTLk7284fS507iVp8HecsWX1S9Kesrb0yq+dMaNweFsfyBssZfWTtnwqNdZE1rnkLMsbHCz7505/DI1vkpO/qd45ptAYXF8tbV3hHE+y30nzBaReA2LHVUlKz3fKG34+YhjpkBNd0vei7O+R450k5R/ujBFbV8bmz+rrtOd4+65YL+2KGru2rpL6Hh27TtHxzpgVylO/y3ldcKSUUeDUc69+sWX4epEk67TtNa845zyzb1s9HHmW1HdE7Hpfveucv3DZRdIxk53Xbu28udGpo/Djsta5JjfVOdecUNtKSpaKgv2m8Ginz+/Nzn/Ens8kn9TS1Pa+cqPT9kJtQIpsd+0dX/9RzriWURCZHmrP4fIOk7IK3c9DaByWnGvf8hlSS3Pb8pQMZzyrKXeOKSSQ7bT/hurI62NmoXMda6x1xnxJ+maFk8+fLvU/ti2vSXb6UXTfSPJJgRyn/bY0SRuXBI8j2Gclpz7Cx9N4GuukLS6/1h3eRyX3/pIzyBlzwq8pIXmHOn2mI775TGoIfibZuERa/4Ezhmf0dj6HDb/JGTtCUtKdz0HhGvY4fddE/b98sl/ypba9b2506iWcSWq7RuxNS3PkONTS1PaZITlFaml0+klKhiTjpDc1RLaZ1nVbYtuiMcF1D17d60cG//6W9Owk7/ebf7j0s2WRaU9NdD4Uh/vBzLYPIJK0e5v0+yNjt3f+Q9LcG5yBd29GTpYu/LP07m+lD9qJejMLnQ9l0a54RTr01Nj0sr9LDx/X/r5/8mHbRWjmFc4Hs57kqHOkL147MNsaPUU6/0Hn9YOj3T8EhVw+Rzr8jNj0d+6SPvyvzu+79Abpe/dIM6dIa15yz3Ppc9KQCZ3fdrT5/0f66/2x6f2OlX781/jrzfmxtOL5+Mv//VNp1QvSe7/dv/L9emtkcPj/Lpa+ets97/HXSBN+57x+Yry0YdH+7TskJVO6bXPb+4UPSW/dHpnHJEt37oxdd8lj0ryoX7wdcq506TMd2/ezk52AoT0TpkmHni79cUzHthktkC1d9hfpf8fv2/rhTrxeOus+5/VzP9j3/lhwpLOtuf8eu2zYBdIlM5zXfzxe2vFFbJ7kVOmO7Z3f7+yrpVWzI9MmTneCxce+G5v/+B9LS/4Ute8U6Y5gEPf2ndJHD+x9v8kp0i3rpf/oHz/POb+XjvvX2PTf9pGa62PTQ4pOkK5+Kzb9rmz3/HcF/8PMrZ3vq6veigzg43njVmnx9Mi0tNzIANxNeLsLiXd8Fz8hHX1RZNqjpzgBcLjzH5JGX+F+Ho48W/pBcPz7fJ70/GXtl68jfrpIKlsrzb6q/Xyp2dK438b2jbQ85zq0clb8dW/d3PYfkO2p3Cg9EB0gS7r5H1JGftv7F/41dn9n/EY6pFR64izFOO+/pTFX7n3/kvTn090DoKC1Z83U0EP6SH8+rWPb2x+hPhFm6tSpKioq0nXXXSc17NFdt/5SmRnp+smPLtbEf/mFKqp2q7GpSff+6lpNPMsp433//bhmzHpVfQryVNS/UGNOOEk33fqbiO2u+2KNfnjZZO2prdPEcafqgcefVfW6JarOLNbEiRNVUVGhxsZG3XvvvZo4caLWr1+v8ePHa8yYMVq+fLmGDx+up556Sunp6Zo6dapeeeUV+Xw+jRs3rlPftPAjgwAAAMBBYvLkyZo5c2br+5lz39bk88YpkJqiF//n91r+5rOaP+tPuvGeP8haq2Ur1uj5V97Sp28/p3lPP6hPPnP5ZkjSDb+8STf86w+08t2ZGtiv7a6dQCCgF198UcuXL9f8+fN14403KvRlwxdffKFrr71Wa9euVa9evTR9+nSVl5frxRdf1OrVq7VixQrdfvsB+o+EOAg+AAAAgC4yatQobd++XVu2bNFnK1YqN7uXigb0lbVWt93/R4088xKdOfmn2ry1TNvKyvXBx3/T98efrvS0NPXKytT533O5g0XSosUfa9K5Z0qSfvD9tm+jrbW67bbbNHLkSJ155pnavHmztm1z7o4pKipSaWmpJOnyyy/Xhx9+qOzsbAUCAV199dWaM2eO0tM7eAvZPupecz4AAACAbmbSpEmaPXu2tm7eqMnnj5MkPTPndZWVV2jZ68/I7/er+IRzVFffsN/7euaZZ1RWVqZly5Y52y0uVl2dM6/EGBOR1xgjn8+nJUuW6N1339Xs2bP1xz/+Ue+9995+lyOe7hV8pOU4k0rdbFjkzHvIOzTy3u2BxzkTq0JpofVD73MGORMVw7cTvY/w5SF9hsVOOArkRL5P9ruXN6OPkx4+5+Obz2InDUnOfBPJmZgdXfbw40nLc79fOt6kPX/AWdftPve0XGeCcvgk0wKXuSv7Inqf8cqwL5L8zkStjkjJdCaW1+50JoY2VMfPG16HVZulbJf2kHdo2+v+x0qZfZzXoWNLyWybPxPdTkJCDw9orxzR7Th8vXh1NOhE9wnu+yJ7oPMQgdDDBkLcJqiGyz9cGlAS+wCG0HH4As623frLhkVOHwwdZ7z2UjQ2dqJgnyHx6zY3bMJp4fD2t53Vz5m0KTkTO6OPX4o8luh1o4/LxJnEmdU3Nm/4RNm96X1U5EMr3MaKrL5t/T9a2edO33dbP2TgcZGT0qPtrY9HtN1Dwsoe7I/x6iC0XtkXTr7CEW33o2cXOX3O7ZjC+0W/Y9wfvBA+KbYzCo6I3WdGb6e/u5Ultzg2PXwicc6g2OVu16Qkn9PW3a4JkpOeGWei7qATnAmz8c5zqC9Eyx4kVW1wXya5t/PQAww6OsaH1u/IPAPJ/XymZjn7DXEdMw9RDLfjyxnkPLAgWr+RsRN6M4Jjvtt5CO/D8cbiPsOl7e631rjypzlla+9zgeSU061vpPaS8oPtt7HG+QwSvX70eBqPL9W9vUdPVs936S+9Bjh15rZ+9MN/2tN3RNuk8HjtLcGTsCdPnqx/+7d/046yMv31xf+VUjJUVdOgPgV58vv9mv/RJ/p60zeSpO+MHa0rf3G3bv3l9WpqbtLcdz7Uj68ZGrPNsSccrxfe/EiTvz9Bz7/2qiQjpaSrqqpKffr0cbY7f76+/rrtIQgbNmzQokWLdOKJJ+rZZ5/VySefrOrqatXU1GjChAkqLS3VoYceGrOvA6l7TTgHAAAAOsFtQnQijBgxQgUFBZo/f74kaceOHTrvvPNUXV2tkpISLV68WK+//rqKi4t13333acaMGerTp48GDRqk0aNH66abIh9I8uWXX+ryyy9XbW2txo8fr2eeeUabN2+Ou11JGj9+vEpKSrRs2TINGzZMTz/9tKqqqjRx4kTV1dXJWqubbrpJU6ZM6fBxdXbCOcEHAAAAeqyDJfg40GpqapSWliZjjJ5//nk999xzevnl+E8mXb9+vc4991ytWrXqgJajs8FH97rtCgAAAICWLVum66+/XtZa5eTk6Iknnkh0kTqE4AMAAADoZk455RR99tlne88YVFxcfMC/9dgXPGoXAAAAgCcIPgAAAAB4guADAAAAgCcIPgAAAAB4guADAAAASIDi4mLt2LFDknTSSScluDTeIPgAAAAAEmzhwoWJLoIneNQuAAAAerwRM0Z0+T5WTlnpmn7BBRdo48aNqqur0w033KBrrrkmJk9mZqaqq6vV0tKi66+/Xu+9956Kiork9/t11VVX6eKLL1ZxcbGmTJmiuXPnqrGxUbNmzdKQIUMitlNTU6Mrr7xSq1at0lFHHaUtW7bo4YcfVklJiX7605/qk08+UW1trS6++GLdfffdkpxvYC655BK9/vrrSktL07PPPqvDDz9cs2bN0t13363k5GRlZ2fr/fff3+9zRPABAAAAdKEnnnhCeXl5qq2t1XHHHaeLLrpI+fn5rnnnzJmj9evXa82aNdq+fbuGDh2qq666qnV5QUGBli9frunTp2vatGl6/PHHI9afPn26cnNztWbNGq1atUrHHnts67L77rtPeXl5am5u1hlnnKEVK1Zo5MiRkqTs7GytXLlSTz31lH7+85/r1Vdf1T333KM333xTAwYMUGVl5QE5F9x2BQAAAHShBx98UMccc4zGjh2rjRs36ssvv4yb98MPP9SkSZOUlJSkvn376vTTT49YfuGFF0qSxowZo/Xr17uuf+mll0qSjj766NbgQpJmzpyp0aNHa9SoUVq9erXWrFnTuuyyyy5r/bto0SJJUmlpqa688ko99thjam5u3reDj8I3HwAAAEAXWbBggd555x0tWrRI6enpOu2001RXV7fP20tNTZUkJScnq6mpqcPr/fOf/9S0adP0ySefKDc3V1deeWVEOYwxMa8fffRRffzxx3rttdc0ZswYLVu2LO43Nh3FNx8AAABAF6mqqlJubq7S09P1+eefa/Hixe3mLy0t1QsvvKCWlhZt27ZNCxYs6NT+SktLNXPmTEnSmjVrtHKlMw9l165dysjIUHZ2trZt26bXX389Yr2//OUvrX9PPPFESdK6det0wgkn6J577lHv3r21cePGTpXFDd98AAAAoMeLNxm8q40fP16PPvqohg4dqqOOOkpjx45tN/9FF12kd999V8OGDVNRUZFGjx6t7OzsDu/v2muv1ZQpUzRs2DANGTJEw4cPV3Z2to444giNGjVKQ4YMUVFRkUpLSyPWq6io0MiRI5WamqrnnntOknTzzTfryy+/lLVWZ5xxho455pjOn4Aoxlrb4cwlJSV26dKl+71TAAAAwAtr167V0KFDE12MTqmurlZmZqbKy8t1/PHH66OPPlLfvn07tG5zc7MaGxsVCAS0bt06nXnmmfriiy+UkpISd53i4mItXbpUBQUFnS6r2/k1xiyz1pa45eebDwAAAOAgcu6556qyslINDQ264447Ohx4SM6jdk8//XQ1NjbKWqvp06e3G3h4jeADAAAAOIh0dp5HuKysLHX2TiW3p2Z1FSacAwAAoEfrzDQDdNy+nFeCDwAAAPRYgUBA5eXlBCAHmLVW5eXlCgQCnVqP264AAADQYw0cOFCbNm1SWVlZoovS4wQCAQ0cOLBT6xB8AAAAoMfy+/0aPHhwoouBIG67AgAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJgg8AAAAAniD4AAAAAOAJX6IL0BmNzY0qryt3XbancY+STbICvoAkaeuerZKkgrQC+ZJ8qm6oVrNtVnZqdus65XXlyk3NVZKJjMH6ZvTda1lC2w+XnZqtNF9aTPruht3a07gnIm1H7Q41tTTFrFPXVKfK+srW9/5kvwrTC1WQVqCaxhrtatglSWpsaVR5bbn8SX7lp+VrZ91ONTQ3qE96n9bjyUrJUoY/I2K9EF+STwVpBTFlLa8tV2NLY0RaVkqWjEzENnbW7VRzS7OyUrJUWV+piroKtahFjc2NSklOkS/JJ5/xKd2fHlGm/LR8+ZP8MfttbmlWWW1ZRNrWPVtb66+mqUa76nepd3pvJZvkdo9hX0Ufe3lduRqbG5UbyFVqcmrc9bbXbJcvyaeAL6Bd9c45Ch1nqH46ItS+o9t5fiBfAV9AvdN7R+SPPmehc5Tpz1RmSmZE3lA7c2sLbmWsb65XRV1FTBnzAnlKSU7p0PG4qairUH1zfUx6flq+rLXaWbfTdb365vqIOqioq1B2anZM3w21lxC34w23p3GPMvwZre8bWxpV11QnK6uaxprW9LxAnnICOeqV0iti/VA/rKqvUm1TbWt634y+yvBnKCslK+6+3YTGhXDR9RPvmPpm9I1YFhoTQkLnJl654m03Xp91K2uoT+5q2BVx/iSnP0fvOzU5VbmB3NblkiLOZbo/Xb1SeslaK2OMpNgxMtQmC9IKtKN2R0w5U5JTlBfIk+TU9+6G3ZKkhuaG1va2dc/W1nEqJzVHAV8g7ngeYq3VtpptMemhthp+jUj3p6u5pVm+JF/E8Seb5Jh+3RnRddye0HGFr1NZX6m6pjoFfAHlpOZIch8P2ttXvOtli23R9prtkiLPtS/J19pumm1z63gevq1QWfdm255tsrIRaeFtqrPijU+903orOSnZZQ1HbVOtquqrJLWNCVJk3wl/7fb5Id51JjTOR48xeYE8pSanth5/aLlbm+qd1ltltWUxxxc634Xpha39y6185bXlSvOlKd2fLknaVrNNGb4MZaZktl6zwvmT/coP5MeMHaFjiS5Hflq+9jTuaW2DIaHPMG7nI1q8a1O8/L1SesWMd1X1VcpKyYq5rkiR7Ty8vkPcPo+U1ZSp2TZHtIleKb2U7k9XWU2ZMvwZCvgCrf1EktJ8aWq2zTGf5+Kdj+6qWwUfX1Z+qcmvTu7SfRgZrZiyYq/5zplzjhpaIgfi//zOf+rswWfH5J2xeob+tOJP+1ymUX1G6amzn9LcdXN178f3dni9m0tu1hXDr9CCjQt0ywe3RCw7IvcIzTl/Tsw6t35wqxZ9sygi7bYTblOySdZvF/92n8of7qWJL+mwnMNi0qsaqvS92d/r1LZG9h6pZyY8s99lCpn6wVQt/mZxu3nCL5wdMbbfWD027rEO5f2q8itd8uolrssOzzlcL058MSKtvK68w+ds2qnTdFbxWXp3w7u67cPbIpad0O8EPT7u8Yi0FWUrdNWbV8Vs59kJz2pE7xEd2qebOxfeqfkb58ekv3zBy6qsq9SUN6bs87Yl6e2L3464SMzfOF9TP5i6X9sMuXL4lbqx5MaItPVV63XhKxe65r9m5DX62aifdWof175zrdbuXBuRNqZwjJ4c/2Tr+1f/8WpMX0zzpWnJD5e41m+0H4/8sa4fdX1Merzx5bXvv6ZBvQZ1qKyhceXJVU/qsZV7b/ffO+R7+sNpf5Aknf/S+REfrjpr3vfnacKLE2LSTys6TQ999yFJ0ktfvaT7l9zfoe09cNoDOuOQM+Iub2hp6PSYFW1w9mC9csEr+7z+7R/erg82fxCTnuZLizmXj575qEoHlOqOj+7Q+5vej7vN0X1Ga8bZM2LS71x4pxZsXBCznyU/XOK6ndqm2n0+P6Gy7s2kuZNUUR/5nyTji8frd6f+bp/2e9fCu/Texvdi0hdcsqDd/0RauGWhfj7/5+1ue+4Fc1WcXSxJGjd7XEzQ9Kcz/6STBpwUs151Y3Xc8xgv4A4XSA7orYvfarcuPv3RpxFB4IQ5E2L+E3JfhB+zJFXUV3SqTdxUcpOmDI+8Juxq2OW6jRnjZ2h04eiY9Mr6Stf8959yf4evDVkpWVp42cLW9ws2LtCv3v9VRJ6heUM187yZEWlXvXmV1u9a36F9dMSvjvuVfjTsRwdse4nEbVcAAAAAPEHwAQAAAMATBB8AAAAAPNGt5nz4knwqTC90XRaa+BdaHnqfZJLUO613zPJQnnRfesQEwPBJV+0pzCiMmXwXb1Jypj8zptzhExWjyxQtNFkyzZ8Wc3yh9UPvs1KylO5zJoWFJocFfIGY/ecH3O9fzQ3kxuRN96UrySTttZzxhJ/j8PtKwyUpKe45Cj8+n/G13nsbOi8HSvSxu9WR25yPeOeiML2wUxMfQ+3bbXtu9xu3VyfR5zKQ7EzedGsLeamx5zElOcW1r/mTYyced0ZOao7rdn3GJ3+Sv93+HX2sGf4MZfojJ9ZHty+3493bduOJ3pckJSclu9ZZYXqha/69yU/Lj62fqHae5kuLrd/g5NzwZW7H0l65wseXcPEm2ual5cUdVzJT4o954enhDwApTC9UbVOt67lsbGlsnbga77iSkmLHEEnKTW3rg+m+9HbPT/j2Un3xHzIRni9aqE21Ny6E7O8DM9zGa8l9zkdoMm54H3QbM+KNq7mpsftqb1K4kenQuQ4Xyt/Rh1r0Tu8dkze8TXVWTsB9fHKbgBwukByIe6yh9PB+VJhRKGsj53zEG1uNMXHbU++0tgewxBv/U5NTW68V8coWLdTnQuJ9vtpbO48eO0JtoiN9Q2r7DBMu+roX4vZQjPbyu10b3D4TSop5n5qcGjv2uVyj89Py445pobTs1OyYyesh0de40Ge7nsBEd4D2lJSU2KVLl3ZhcQAAAAB0Z8aYZdbaErdl3HYFAAAAwBMEHwAAAAA8QfABAAAAwBMEHwAAAAA80akJ58aYMklfd11xOqVAUvs/7YnujPrt+ajjno367dmo356N+u3ZvKjfQ6y1vd0WdCr4OJgYY5bGm0WP7o/67fmo456N+u3ZqN+ejfrt2RJdv9x2BQAAAMATBB8AAAAAPNGdg48/J7oA6FLUb89HHfds1G/PRv32bNRvz5bQ+u22cz4AAAAAdC/d+ZsPAAAAAN1Itww+jDHjjTFfGGO+MsZMTXR5sP+MMeuNMSuNMZ8aY5YG0/KMMW8bY74M/s1NdDnRMcaYJ4wx240xq8LSXOvTOB4M9ucVxpjRiSs5OiJO/d5ljNkc7MOfGmMmhC27NVi/XxhjzkpMqdFRxpgiY8x8Y8waY8xqY8wNwXT6cA/QTv3Sh3sIY0zAGLPEGPNZsI7vDqYPNsZ8HKzLvxhjUoLpqcH3XwWXF3dl+bpd8GGMSZb0sKSzJQ2TdJkxZlhiS4UD5HRr7bFhj3+bKulda+0Rkt4Nvkf38KSk8VFp8erzbElHBP9dI+kRj8qIffekYutXkv4r2IePtdbOk6Tg+HyppOHBdaYHx3EcvJok3WitHSZprKTrgvVIH+4Z4tWvRB/uKeolfddae4ykYyWNN8aMlfR/5dTx4ZIqJF0dzH+1pIpg+n8F83WZbhd8SDpe0lfW2n9YaxskPS9pYoLLhK4xUdKM4OsZki5IXFHQGdba9yXtjEqOV58TJT1lHYsl5Rhj+nlSUOyTOPUbz0RJz1tr6621/5T0lZxxHAcpa+031trlwde7Ja2VNED04R6hnfqNhz7czQT7YnXwrT/4z0r6rqTZwfToPhzq27MlnWGMMV1Vvu4YfAyQtDHs/Sa132nQPVhJbxljlhljrgmmFVprvwm+3iqpMDFFwwESrz7p0z3H9cHbbp4Iu02S+u3GgrdfjJL0sejDPU5U/Ur04R7DGJNsjPlU0nZJb0taJ6nSWtsUzBJej611HFxeJSm/q8rWHYMP9EwnW2tHy/n6/jpjzHfCF1rnsWw8mq2HoD57pEckHSbnK/5vJP0+oaXBfjPGZEp6QdLPrbW7wpfRh7s/l/qlD/cg1tpma+2xkgbK+aZqSGJL1KY7Bh+bJRWFvR8YTEM3Zq3dHPy7XdKLcjrKttBX98G/2xNXQhwA8eqTPt0DWGu3BS92LZIeU9ttGdRvN2SM8cv5YPqMtXZOMJk+3EO41S99uGey1lZKmi/pRDm3RPqCi8LrsbWOg8uzJZV3VZm6Y/DxiaQjgjP2U+RMgnolwWXCfjDGZBhjskKvJY2TtEpOvU4JZpsi6eXElBAHSLz6fEXSFcEn5oyVVBV2awe6iah7/L8vpw9LTv1eGnyaymA5k5KXeF0+dFzwXu//kbTWWvuHsEX04R4gXv3Sh3sOY0xvY0xO8HWapO/JmdszX9LFwWzRfTjUty+W9J7twh8C9O09y8HFWttkjLle0puSkiU9Ya1dneBiYf8USnoxOLfJJ+lZa+0bxphPJM00xlwt6WtJlySwjOgEY8xzkk6TVGCM2STpTkn3y70+50maIGcSY42kf/G8wOiUOPV7mjHmWDm34qyX9GNJstauNsbMlLRGzlN2rrPWNieg2Oi4Ukk/krQyeM+4JN0m+nBPEa9+L6MP9xj9JM0IPpUsSdJMa+2rxpg1kp43xtwr6W9yglAF/z5tjPlKzsNELu3KwvEL5wAAAAA80R1vuwIAAADQDRF8AAAAAPAEwQcAAAAATxB8AAAAAPAEwQcAAAAATxB8AAAAAPAEwQcAfMsYY3KMMdeGve9vjJndRfu6wBjzm3aWjzDGPNkV+wYAHHz4nQ8A+JYxxhRLetVae7QH+1oo6Xxr7Y528rwj6Spr7YauLg8AILH45gMAvn3ul3SYMeZTY8zvjDHFxphVkmSMudIY85Ix5m1jzHpjzPXGmF8aY/5mjFlsjMkL5jvMGPOGMWaZMeYDY8yQ6J0YY46UVB8KPIwxk4wxq4wxnxlj3g/LOldd/Iu6AICDA8EHAHz7TJW0zlp7rLX2ZpflR0u6UNJxku6TVGOtHSVpkaQrgnn+LOln1toxkm6SNN1lO6WSloe9/42ks6y1x0g6Pyx9qaRT9uN4AADdhC/RBQAAHHTmW2t3S9ptjKmS882EJK2UNNIYkynpJEmzjDGhdVJdttNPUlnY+48kPWmMmSlpTlj6dkn9D2D5AQAHKYIPAEC0+rDXLWHvW+RcN5IkVVprj93LdmolZYfeWGt/Yow5QdI5kpYZY8ZYa8slBYJ5AQA9HLddAcC3z25JWfu6srV2l6R/GmMmSZJxHOOSda2kw0NvjDGHWWs/ttb+Rs43IkXBRUdKWrWv5QEAdB8EHwDwLRP8tuGj4OTv3+3jZn4o6WpjzGeSVkua6JLnfUmjTNu9Wb8zxqwMTm5fKOmzYPrpkl7bx3IAALoRHrULAOgyxpj/ljTXWvtOnOWpkv4q6WRrbZOnhQMAeI5vPgAAXek/JKW3s3yQpKkEHgDw7cA3HwAAAAA8wTcfAAAAADxB8AEAAADAEwQfAAAAADxB8AEAAADAEwQfAAAAADzx/wFuIEVmInBJigAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "min_x = 0\n", "max_x = audio.duration_s\n", "\n", "x_audio = np.arange(min_x, max_x + 0.01, 0.01)\n", "\n", "y_slice = np.array([np.nan] * len(x_audio))\n", "for start_s, end_s in slice_intervals:\n", " y_slice[int(round(start_s*100)):int(round(end_s*100))] = 1\n", " \n", "y_vad = np.array([np.nan] * len(x_audio))\n", "for e in vad_gaps:\n", " y_vad[int(round(e.lower*100)):int(round(e.upper*100))] = 0.9\n", " \n", "y_align = np.array([np.nan] * len(x_audio))\n", "for e in align_gaps:\n", " y_align[int(round(e.lower*100)):int(round(e.upper*100))] = 0.8\n", "\n", " \n", "plt.figure(figsize=(14, 3))\n", "plt.plot(x_audio, y_slice, linewidth=5, label=\"slices\")\n", "plt.plot(x_audio, y_vad, linewidth=5, label=\"vad gaps\")\n", "plt.plot(x_audio, y_align, linewidth=5, label=\"align gaps\")\n", "gca().axes.get_yaxis().set_visible(False)\n", "plt.legend()\n", "plt.xlabel(\"time (s)\")\n", "plt.show();" ] }, { "cell_type": "code", "execution_count": null, "id": "554a3576", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b34c6371", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f41f79ca", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "091c12f9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "7ce82fd1", "metadata": {}, "source": [ "## multi gpu nemo pred" ] }, { "cell_type": "code", "execution_count": 1, "id": "c4af4fe9", "metadata": {}, "outputs": [], "source": [ "import json\n", "with open(\"/home/mikey/notebooks/datasets/mixed-sample-metadata_big.json\") as f:\n", " data = json.load(f)" ] }, { "cell_type": "code", "execution_count": 2, "id": "07d46ef9", "metadata": {}, "outputs": [], "source": [ "from suno_utils.web.nemo import transcribe_multi_gpu\n", "filepaths = [e[\"uri\"] for e in data[:200]]\n", "out = transcribe_multi_gpu(filepaths, quiet=False)" ] }, { "cell_type": "code", "execution_count": 3, "id": "6891adaf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "200\n" ] }, { "data": { "text/plain": [ "['arthur witney went on to create the k language a proprietary array language',\n", " 'lasers pointed at aircraft are a hazard to aviation']" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "print(len(out))\n", "out[:2]" ] }, { "cell_type": "code", "execution_count": null, "id": "cf8267b7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f9272038", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e78b3a53", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "c88d5747", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 80, "id": "919508a2", "metadata": {}, "outputs": [], "source": [ "with open(\"transcribeMe_1139063__4bab9d63-c1ce-4c65-be67-75bbc711ccb4__0__300000.json\") as f:\n", " d = json.load(f)\n" ] }, { "cell_type": "code", "execution_count": 84, "id": "ed46a4c2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "dict_keys(['speaker', 'words'])" ] }, "execution_count": 84, "metadata": {}, "output_type": "execute_result" } ], "source": [ "d[\"text\"][0].keys()" ] }, { "cell_type": "code", "execution_count": 91, "id": "73bc5221", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Hello. My name is Barry Williame Magliarditi, your host of this episode of the Freedom Series, uh, livestream session, and today, I've got Ronan Leonard on the phone, uh, on the call from lockdown in Sicktoria, Melbourne. Mate, how you doing today?\n", "\n", "Very well, thanks. Well, today is either yesterday, today, or tomorrow, right? There are no weekends. It's just one of those three days. [laughter] It's like--\n", "\n", "Yeah.\n", "\n", "--déjà vu all over again.\n", "\n", "Yep. Absolutely. Absolutely. Well, I, I guess, uh, super excited for us to chat today, and we're gonna chat to-- so Ronan's a cofounder of a disruptive management consulting company that helps organizations get a three to seven times increase in engagement by identifying the unmet needs of their customers. Uh, and what I'd like us to kinda dive into a little bit today is talk about the hypernormalization of low engagement due to mass communication tools, which have sacrificed efficiency for relevancy. And before we do, uh, I'd like to just, just extend a warm welcome to everyone who's watching us live. Wherever you are in the world, whatever platform you're on, um, welcome to the show. [Raffi?] from my team, uh, hey, [Raffi?]. How you doing, mate? He's out there, uh, overseeing all your comments and questions. So let's make this as-- uh, I guess as conversational as possible. So any questions you've got for myself or Ronan, please put them in the chat of wherever you're watching this, and, uh, we'll get them answered here. Hit the like button. Let us know that you're watching. And also, uh, why not start a watch party and share it across your feed to other business owners and entrepreneurs as well? So, Ronan, let's talk a little bit about the hypernormalization. Where do we kick off with this?\n", "\n", "We probably kick off with the fact that everybody would agree that it's costing them far more to reach fewer people on just about every social media platform.\n", "\n", "Absolutely.\n", "\n", "So what they tend to do is just produce more content. Uh, you know, Gary V's telling small business owners to film the day in the life and, and chop up all this content [inaudible]. Let's be honest. Who's going to watch it? If you don't-- if you haven't already built a, uh, a following of, of people across these social media platforms, just pushing more content out there isn't gonna get you more engagement. So there's a couple of things within that. First off, we, we-- as I say, communication tools become the norms. Everyone builds a funnel. Everybody, uh, creates a piece of content and pushes it out to four or five platforms at the same time, and we exponentially are creating more and more content but with fewer and fewer engagement. At the same time, we're kinda waving the white flag and going, \"Okay. Well, if, if my conversion rate's only 3%, then I've just gotta put more people into a funnel and, and then another 100 people, another 100 people in to get my 3 people to come out of that--\"\n", "\n", "Yeah.\n", "\n", "--without ever really addressing the root cause of that, which is you probably don't know your customers well enough. You don't know what they're thinking. You don't know their unmet needs. You don't know what they're really worried or concerned about, their beliefs and their concerns. And therefore, you just think, \"Well, it's okay. There's a-- there's a never-ending stream of customers out there,\" eh, which is fine for a very small business, right? If you've got a very small business and you've got a huge target market, uh, you go, \"Oh, that's not a problem.\" But you r-reach this point where you've saturated the market. Everyone's either heard of you or not interested 'cause, again, you don't know what they're really interested in or, as I said, it just gets more and more expensive to put people into that, that funnel or your marketing sequence. So would you agree with that, Barry? What are you-- what are your thoughts on that?\n", "\n", "Yeah. Well, I-- yes. It's a-- it's a very interesting topic, and, um, you know, we're seeing-- like, obviously, Google platform's quite expensive to, to advertise on. But, you know, you're speaking to a target audience. If you look at Eugene Schwartz's five levels of awareness, you go from, you know, unaware through to problem aware--\n", "\n", "Mm-hmm.\n", "\n", "--solution aware, product aware, most aware. You know, you speak to someone on Google if you're advertising there who's product or most aware. So therefore, you are paying more for a click or for a lead, um, but you're also speaking to a more qualified buyer. Now, the problem with that is is that you are competing against, um, other companies like for like, or perceivedly like for like, whereas advertising on social platforms like Facebook or Instagram where people aren't going there to look up your products or service, if you're able to present a piece of, you know, content or an ad or a funnel, um, something that buys their attention, you can get a much cheaper click because you're reaching someone who's either, like, problem or solution aware.\n", "\n", "Yeah.\n", "\n", "You've got a chance to influence them more. But I think you're right because, you know, I see many business owners start with-- entrepreneurs start with going after a mass target market. Now, in the beginning, it's okay. They're trying to be everything to everyone, and they're trying to find, like, message-to-market fit. Yet there does become a point in business where that needs to change. And this is where I see a challenge that a lot of entrepreneurs face is they're too rigid in their approach to growing business. It's like they start off doing something, and they get some form of result, so they keep just doing all that same stuff without even--\n", "\n", "Mm-hmm.\n", "\n", "--challenging whether or not what they're doing is effective. And as you said, it's like they produce content, but do they have the metrics in place to measure which piece of content is actually piquing the curiosity of their audience? Which piece of content is actually gonna [inaudible]?\n", "\n" ] } ], "source": [ "for s in d[\"text\"]:\n", " print(\" \".join([t[\"name\"] for t in s[\"words\"]]))\n", " print()" ] }, { "cell_type": "code", "execution_count": 87, "id": "5b7a7d03", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'name': 'Hello.'}" ] }, "execution_count": 87, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s[\"words\"][0]" ] }, { "cell_type": "code", "execution_count": 88, "id": "e71e4632", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[{'name': 'Hello.'},\n", " {'name': 'My'},\n", " {'name': 'name'},\n", " {'name': 'is'},\n", " {'name': 'Barry'},\n", " {'name': 'Williame'},\n", " {'name': 'Magliarditi,'},\n", " {'name': 'your'},\n", " {'name': 'host'},\n", " {'name': 'of'},\n", " {'name': 'this'},\n", " {'name': 'episode'},\n", " {'name': 'of'},\n", " {'name': 'the'},\n", " {'name': 'Freedom'},\n", " {'name': 'Series,'},\n", " {'name': 'uh,'},\n", " {'name': 'livestream'},\n", " {'name': 'session,'},\n", " {'name': 'and'},\n", " {'name': 'today,'},\n", " {'name': \"I've\"},\n", " {'name': 'got'},\n", " {'name': 'Ronan'},\n", " {'name': 'Leonard'},\n", " {'name': 'on'},\n", " {'name': 'the'},\n", " {'name': 'phone,'},\n", " {'name': 'uh,'},\n", " {'name': 'on'},\n", " {'name': 'the'},\n", " {'name': 'call'},\n", " {'name': 'from'},\n", " {'name': 'lockdown'},\n", " {'name': 'in'},\n", " {'name': 'Sicktoria,'},\n", " {'name': 'Melbourne.'},\n", " {'name': 'Mate,'},\n", " {'name': 'how'},\n", " {'name': 'you'},\n", " {'name': 'doing'},\n", " {'name': 'today?'}]" ] }, "execution_count": 88, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "fc46e107", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a486092c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c94958ce", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "31a212cd", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "37f12b05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c36f53a7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ede8f0af", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "044bf70a", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }