{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "47a1e830",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import json\n",
"import tqdm\n",
"\n",
"import pandas as pd\n",
"from multiprocessing import Pool\n",
"\n",
"from suno_utils.harvest.youtube.constants_languages import USED_LANG_CODES\n",
"\n",
"\n",
"DATA_DIR = \"/data2/suno/data/harvest/youtube\""
]
},
{
"cell_type": "markdown",
"id": "ac6ddc2b",
"metadata": {},
"source": [
"## get all unique vtt files and do parsing into new file"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "70e33ca1",
"metadata": {},
"outputs": [],
"source": [
"# !wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "d72f7a4a",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
]
}
],
"source": [
"import fasttext\n",
"\n",
"from suno_utils.harvest.youtube.captions import parse_vtt, verify_vtt_lang\n",
"\n",
"fasttext_lang_model = fasttext.load_model(\"lid.176.bin\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b765c11a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2280733"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vtt_fns = [fn for fn in os.listdir(os.path.join(DATA_DIR, \"audio\")) if fn.endswith(\"vtt\")]\n",
"vtt_fns = [fn for fn in vtt_fns if len(fn.split(\".\")) == 3]\n",
"id_list = [fn.split(\".\")[0] for fn in vtt_fns]\n",
"s = pd.Series(id_list).value_counts()\n",
"unique_ids = set(s[s==1].index)\n",
"filtered_vtt_fns = [\n",
" vtt_fn \n",
" for vtt_fn in vtt_fns \n",
" if (\n",
" len(vtt_fn.split(\".\")[1]) >= 2 and \n",
" vtt_fn.split(\".\")[0] in unique_ids\n",
" )\n",
"]\n",
"len(filtered_vtt_fns)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "631a0cf2",
"metadata": {},
"outputs": [],
"source": [
"def foo(vtt_fn):\n",
" with open(os.path.join(DATA_DIR, \"audio\", vtt_fn)) as f:\n",
" vtt_str = f.read()\n",
" try:\n",
" check_lang_code = vtt_fn.split(\".\")[1]\n",
" _, annotated_lines = parse_vtt(vtt_str)\n",
" verify_vtt_lang(annotated_lines, check_lang_code, fasttext_lang_model)\n",
" verified_vtt_fns.append(vtt_fn)\n",
" return vtt_fn\n",
" except:\n",
" pass\n",
" return None\n",
"\n",
"with Pool(30) as p:\n",
" out = p.map(foo, filtered_vtt_fns, chunksize=1)"
]
},
{
"cell_type": "code",
"execution_count": 43,
"id": "d3fa84c8",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"1815144"
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"verified_vtt_fns = [e for e in out if e is not None]\n",
"len(verified_vtt_fns)"
]
},
{
"cell_type": "code",
"execution_count": 90,
"id": "1326c974",
"metadata": {},
"outputs": [],
"source": [
"with open(\"verified_vtt_fns.json\", \"w\") as f:\n",
" json.dump(verified_vtt_fns, f)"
]
},
{
"cell_type": "markdown",
"id": "1a45ea91",
"metadata": {},
"source": [
"## Find tags"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b1d3bbdb",
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"\n",
"from collections import defaultdict, Counter\n",
"\n",
"from suno_utils.utils.text import normalize_whitespace"
]
},
{
"cell_type": "code",
"execution_count": 74,
"id": "502e99f8",
"metadata": {},
"outputs": [],
"source": [
"def foo(vtt_fn):\n",
" base_lang = vtt_fn.split(\".\")[1].split(\"-\")[0]\n",
" with open(os.path.join(DATA_DIR, \"audio\", vtt_fn)) as f:\n",
" vtt_str = f.read()\n",
" try:\n",
" _, annotated_lines = parse_vtt(vtt_str)\n",
" text = \"\".join([s for _, s in annotated_lines])\n",
" tags_br = [\n",
" normalize_whitespace(s.lower()) \n",
" for s in re.findall(r\"\\[(.*?)\\]\", text) \n",
" if len(s.strip()) > 0\n",
" ]\n",
" tags_pa = [\n",
" normalize_whitespace(s.lower()) \n",
" for s in re.findall(r\"\\((.*?)\\)\", text) \n",
" if len(s.strip()) > 0\n",
" ]\n",
" if len(tags_br) + len(tags_pa) == 0:\n",
" return None\n",
" return base_lang, list(set(tags_br)), list(set(tags_pa))\n",
" except:\n",
" pass\n",
" return None\n",
"\n",
"with Pool(30) as p:\n",
" out = p.map(foo, verified_vtt_fns, chunksize=1)"
]
},
{
"cell_type": "code",
"execution_count": 85,
"id": "79f447ff",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1304"
]
},
"execution_count": 85,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# keep up 100 per language\n",
"tags_info = defaultdict(list)\n",
"for e in out:\n",
" if e is None:\n",
" continue\n",
" lang, a, b = e\n",
" tags_info[lang].extend(a)\n",
" tags_info[lang].extend(b)\n",
"retained_tags = []\n",
"for k, v in tags_info.items():\n",
" for kk, vv in Counter(v).most_common(100):\n",
" if vv >= 10:\n",
" retained_tags.append(kk)\n",
"len(retained_tags)"
]
},
{
"cell_type": "code",
"execution_count": 89,
"id": "1d9eb360",
"metadata": {},
"outputs": [],
"source": [
"# import json\n",
"# print(json.dumps(retained_tags, indent=4, ensure_ascii=False))"
]
},
{
"cell_type": "markdown",
"id": "d9bd24fa",
"metadata": {},
"source": [
"### do parsing/cleaning/text-filtering base on cap etc"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "b1d63523",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1815144 promising ids\n"
]
}
],
"source": [
"import tqdm\n",
"import funcy\n",
"import re\n",
"import string\n",
"import numpy as np\n",
"\n",
"from suno_utils.harvest.youtube.captions import parse_vtt, clean_cc_lines, validate_quality\n",
"from suno_utils.harvest.youtube.constants_languages import CASED_LANGS, USED_LANG_CODES\n",
"\n",
"with open(\"verified_vtt_fns.json\") as f:\n",
" vtt_fns = json.load(f)\n",
"promising_vtt_fns = set(vtt_fns)\n",
"promising_ids = set([s.split(\".\")[0] for s in vtt_fns])\n",
"print(len(promising_ids), \"promising ids\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "a0132983",
"metadata": {},
"outputs": [],
"source": [
"def _get_cc_name(m):\n",
" cc_name = None\n",
" for k, v in m[\"subtitles\"].items():\n",
" if k.startswith(\"en\"):\n",
" for e in v:\n",
" if e[\"ext\"] == \"vtt\":\n",
" mg = re.search(r\"name\\=([^\\&]+?)\\&\", e[\"url\"])\n",
" if mg is not None:\n",
" cc_name = mg.group(1)\n",
" return cc_name\n",
"\n",
"def process_item(line):\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" return {\"success\": False, \"error_msg\": \"empty\"}\n",
" m = json.loads(line)\n",
" if not m[\"success\"]:\n",
" return {\"success\": False, \"error_msg\": \"harvest issue\"}\n",
" if m[\"id\"] not in promising_ids:\n",
" return {\"success\": False, \"error_msg\": \"file excluded\"}\n",
" youtube_id = m[\"id\"]\n",
" lang_code = list(m[\"meta\"][\"subtitles\"].keys())[0]\n",
" if lang_code not in USED_LANG_CODES:\n",
" return {\"success\": False, \"error_msg\": \"language not supported\"}\n",
" vtt_fn = f\"{youtube_id}.{lang_code}.vtt\"\n",
" audio_fn = m[\"audio_filename\"]\n",
" duration_s = m[\"meta\"][\"duration\"]\n",
" res = {\n",
" \"success\": False, \n",
" \"id\": youtube_id, \n",
" \"lang_code\": lang_code,\n",
" \"vtt_fn\": vtt_fn,\n",
" \"audio_fn\": audio_fn,\n",
" \"duration_s\": duration_s,\n",
" }\n",
" if vtt_fn not in promising_vtt_fns:\n",
" return {\"success\": False, \"error_msg\": \"vtt filename messup\"}\n",
" view_count = m[\"meta\"][\"view_count\"]\n",
" if view_count < 50:\n",
" res[\"error_msg\"] = \"too few views\"\n",
" return res\n",
" if duration_s < 20 or duration_s > 2 * 60 * 60:\n",
" res[\"error_msg\"] = \"too long or short\"\n",
" return res\n",
" try:\n",
" with open(os.path.join(DATA_DIR, \"audio\", vtt_fn)) as f:\n",
" vtt_str = f.read()\n",
" if \"\" in vtt_str or \"\" in vtt_str:\n",
" raise ValueError(\"word based timestamps found. likely auto-generated\")\n",
" lang_code, annotated_lines = parse_vtt(vtt_str)\n",
"# # check captions name\n",
"# if lang_code.split(\"-\")[0] == \"en\":\n",
"# cc_name = _get_cc_name(m[\"meta\"])\n",
"# if cc_name is not None and cc_name != \"Default\":\n",
"# raise ValueError(\"suspicious subtitle name\")\n",
" validate_quality(annotated_lines, lang_code, duration_s=duration_s)\n",
" res[\"annotated_lines\"] = clean_cc_lines(annotated_lines, lang_code=lang_code)\n",
" res[\"success\"] = True\n",
" except Exception as e:\n",
" res[\"error_msg\"] = str(type(e)) + \" -- \" + str(e)\n",
" return res\n",
" return res"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "8c45b686",
"metadata": {},
"outputs": [],
"source": [
"# out = [process_item(line) for line in lines_chunk[:10]]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "5773a89c",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"50it [06:36, 7.93s/it]\n",
"50it [07:31, 9.03s/it]\n",
"205it [1:06:28, 19.46s/it]\n"
]
}
],
"source": [
"n_cores = 31\n",
"chunksize = 10000\n",
"with open(os.path.join(DATA_DIR, \"pipeline\", \"verified_metas.jsonl\"), \"w\") as f:\n",
" f.write(\"\")\n",
"with open(os.path.join(DATA_DIR, \"pipeline\", \"failed_metas.jsonl\"), \"w\") as f:\n",
" f.write(\"\")\n",
"meta_fns = [\"youtube_metas.jsonl\", \"youtube_metas_p2.jsonl\", \"youtube_metas_p3.jsonl\"]\n",
"# tqdm steps: 50, 50, 205\n",
"for meta_fn in meta_fns:\n",
" with open(os.path.join(DATA_DIR, \"pipeline\", \"verified_metas.jsonl\"), \"a\") as fsw:\n",
" with open(os.path.join(DATA_DIR, \"pipeline\", \"failed_metas.jsonl\"), \"a\") as few:\n",
" with open(os.path.join(DATA_DIR, meta_fn)) as f:\n",
" for n, lines_chunk in tqdm.tqdm(\n",
" enumerate(funcy.chunks(chunksize, f.readlines()))\n",
" ):\n",
" with Pool(n_cores) as p:\n",
" out = p.map(process_item, lines_chunk, chunksize=5)\n",
" for m in out:\n",
" if m[\"success\"]:\n",
" fsw.write(json.dumps(m) + \"\\n\")\n",
" else:\n",
" few.write(json.dumps(m) + \"\\n\")\n",
"# break\n",
"# break"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17b5cdfd",
"metadata": {},
"outputs": [],
"source": [
"# TODO:\n",
"# this should prob not be a speaker change: Aos já inscritos - Gratidão pelas inscrições,\n",
"# (3x) for lyrics?\n",
"# repetitions in lyrics - don't merge"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d3c7276",
"metadata": {},
"outputs": [],
"source": [
"# !head -2 /data2/suno/data/harvest/youtube/pipeline/verified_metas.jsonl"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "efd282a3",
"metadata": {},
"outputs": [],
"source": [
"# !head -2 /data2/suno/data/harvest/youtube/pipeline/failed_metas.jsonl"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "71376ae3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "f4d0744b",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "ce213811",
"metadata": {},
"source": [
"### Analyze"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "5925b231",
"metadata": {},
"outputs": [],
"source": [
"# from suno_utils.harvest.youtube.constants_tags import KNOWN_TAGS\n",
"# tags = []\n",
"# with open(\"/data2/suno/data/harvest/youtube/pipeline/verified_metas.jsonl\") as f:\n",
"# for line in f:\n",
"# line = line.strip()\n",
"# if len(line) == 0:\n",
"# continue\n",
"# m = json.loads(line)\n",
"# text_lines = [s for _, s in m[\"annotated_lines\"]]\n",
"# for c in set(\"\".join(text_lines)):\n",
"# if ord(c) >= 983040 and ord(c) <= 1048573:\n",
"# tags.append(c)\n",
"\n",
"# print(len(tags))\n",
"# print([KNOWN_TAGS[ord(e)-983040] for e in tags[:20]])"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "c865ed33",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"997310"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from collections import defaultdict\n",
"lang_durations = defaultdict(list)\n",
"tot_lang_durations = defaultdict(int)\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/verified_metas.jsonl\") as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" lang_durations[m[\"lang_code\"]].append(m[\"duration_s\"])\n",
" tot_lang_durations[m[\"lang_code\"].split(\"-\")[0]] += m[\"duration_s\"] / 60 / 60\n",
"success_langs_vc = {k: len(v) for k, v in lang_durations.items()}\n",
"np.sum(list(success_langs_vc.values()))\n",
"# 997310"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "81008a25",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"150868.1\n",
"(36,)\n"
]
},
{
"data": {
"text/plain": [
"en 103775.3\n",
"ru 6069.7\n",
"es 5066.4\n",
"zh 4217.1\n",
"fr 3181.3\n",
"de 3168.7\n",
"pt 2983.1\n",
"ko 2888.8\n",
"id 2510.9\n",
"ja 2090.5\n",
"tr 2015.7\n",
"pl 1923.2\n",
"hi 1119.7\n",
"it 1019.7\n",
"vi 974.9\n",
"nl 817.2\n",
"sv 655.8\n",
"th 609.6\n",
"el 570.5\n",
"cs 569.7\n",
"fi 551.8\n",
"uk 517.7\n",
"ro 402.5\n",
"hu 378.9\n",
"da 376.4\n",
"sr 329.7\n",
"no 228.0\n",
"sk 182.0\n",
"bg 179.5\n",
"hr 169.0\n",
"fil 141.2\n",
"ms 124.8\n",
"lt 119.8\n",
"ta 113.3\n",
"ca 107.6\n",
"lv 103.5\n",
"sl 99.4\n",
"bn 89.4\n",
"et 71.8\n",
"tlh 58.4\n",
"si 49.2\n",
"kk 47.3\n",
"sq 42.7\n",
"hy 33.9\n",
"bs 28.3\n",
"te 26.6\n",
"az 19.6\n",
"ml 18.1\n",
"mr 14.7\n",
"ne 7.2\n",
"pa 6.0\n",
"kn 1.9\n",
"dtype: float64"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s = pd.Series(tot_lang_durations)\n",
"print(s.sum().round(1))\n",
"print(s[s>=100].shape)\n",
"s.sort_values(ascending=False).round(1)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "434c990c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"729680"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"error_data = defaultdict(list)\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/failed_metas.jsonl\") as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if \"lang_code\" in m:\n",
" error_data[m[\"lang_code\"]].append(m[\"error_msg\"])\n",
"error_langs_vc = {k: len(v) for k, v in error_data.items()}\n",
"np.sum(list(error_langs_vc.values()))\n",
"# 729680"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "fbe74e10",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"[('en', 82282.8, 0.501, 539203, 537333),\n",
" ('en-US', 12520.0, 0.677, 76284, 36341),\n",
" ('en-GB', 7230.1, 0.705, 50977, 21368),\n",
" ('ru', 6069.7, 0.898, 36553, 4149),\n",
" ('es', 3021.4, 0.442, 22061, 27906),\n",
" ('ko', 2888.8, 0.922, 21304, 1807),\n",
" ('fr', 2641.6, 0.611, 19657, 12507),\n",
" ('de', 2628.1, 0.766, 18295, 5587),\n",
" ('id', 2510.9, 0.89, 14385, 1771),\n",
" ('ja', 2090.5, 0.865, 14009, 2179),\n",
" ('tr', 2015.7, 0.899, 11876, 1329),\n",
" ('pl', 1923.2, 0.77, 10476, 3122),\n",
" ('zh-Hans', 1697.7, 0.854, 8668, 1480),\n",
" ('pt-BR', 1482.3, 0.699, 10953, 4713),\n",
" ('pt', 1387.3, 0.464, 10342, 11946),\n",
" ('es-419', 1309.8, 0.632, 10177, 5923),\n",
" ('zh-CN', 1152.0, 0.854, 5196, 887),\n",
" ('zh', 1140.5, 0.887, 6175, 783),\n",
" ('hi', 1119.7, 0.797, 8884, 2256),\n",
" ('it', 1019.7, 0.482, 7999, 8588),\n",
" ('vi', 974.9, 0.882, 6162, 823),\n",
" ('en-IN', 859.9, 0.514, 5971, 5651),\n",
" ('en-CA', 780.0, 0.656, 5166, 2715),\n",
" ('nl', 690.4, 0.623, 7270, 4394),\n",
" ('sv', 655.8, 0.86, 4387, 713),\n",
" ('th', 609.6, 0.89, 3815, 471),\n",
" ('el', 570.5, 0.9, 3305, 369),\n",
" ('cs', 569.7, 0.851, 4671, 819),\n",
" ('fi', 551.8, 0.841, 3002, 567),\n",
" ('uk', 517.7, 0.909, 3532, 354),\n",
" ('de-DE', 468.1, 0.835, 2885, 571),\n",
" ('es-ES', 433.8, 0.684, 3075, 1419),\n",
" ('ro', 402.5, 0.817, 2454, 550),\n",
" ('hu', 378.9, 0.852, 2592, 452),\n",
" ('da', 376.4, 0.788, 4723, 1269),\n",
" ('fr-FR', 286.2, 0.75, 1750, 582),\n",
" ('sr', 263.1, 0.869, 1463, 220),\n",
" ('es-MX', 231.5, 0.198, 1773, 7196),\n",
" ('fr-CA', 230.2, 0.749, 2260, 756),\n",
" ('no', 228.0, 0.846, 2195, 399),\n",
" ('sk', 182.0, 0.884, 1836, 240),\n",
" ('bg', 179.5, 0.84, 1418, 271),\n",
" ('hr', 169.0, 0.863, 1182, 187),\n",
" ('fil', 141.2, 0.84, 1218, 232),\n",
" ('ms', 124.8, 0.853, 763, 131),\n",
" ('zh-TW', 124.2, 0.601, 1088, 721),\n",
" ('lt', 119.8, 0.859, 917, 150),\n",
" ('pt-PT', 113.5, 0.649, 930, 503),\n",
" ('ta', 113.3, 0.637, 796, 453),\n",
" ('ca', 107.6, 0.383, 841, 1353),\n",
" ('lv', 103.5, 0.819, 828, 183),\n",
" ('en-IE', 99.8, 0.621, 686, 419),\n",
" ('sl', 99.4, 0.85, 881, 156),\n",
" ('nl-NL', 93.9, 0.741, 966, 337),\n",
" ('bn', 89.4, 0.738, 885, 314),\n",
" ('et', 71.8, 0.867, 509, 78),\n",
" ('es-US', 69.8, 0.715, 561, 224),\n",
" ('zh-SG', 65.9, 0.922, 331, 28),\n",
" ('sr-Latn', 60.0, 0.892, 397, 48),\n",
" ('tlh', 58.4, 0.656, 103, 54),\n",
" ('si', 49.2, 0.799, 354, 89),\n",
" ('kk', 47.3, 0.86, 416, 68),\n",
" ('de-CH', 44.0, 0.843, 371, 69),\n",
" ('sq', 42.7, 0.833, 504, 101),\n",
" ('zh-HK', 36.8, 0.822, 222, 48),\n",
" ('hy', 33.9, 0.523, 384, 350),\n",
" ('nl-BE', 32.9, 0.695, 376, 165),\n",
" ('de-AT', 28.5, 0.743, 263, 91),\n",
" ('bs', 28.3, 0.91, 162, 16),\n",
" ('te', 26.6, 0.732, 243, 89),\n",
" ('az', 19.6, 0.915, 195, 18),\n",
" ('ml', 18.1, 0.859, 158, 26),\n",
" ('mr', 14.7, 0.933, 111, 8),\n",
" ('fr-BE', 14.5, 0.538, 134, 115),\n",
" ('fr-CH', 8.9, 0.736, 92, 33),\n",
" ('ne', 7.2, 0.846, 88, 16),\n",
" ('sr-Cyrl', 6.6, 0.851, 40, 7),\n",
" ('pa', 6.0, 0.809, 38, 9),\n",
" ('en-AU', 2.8, 0.848, 78, 14),\n",
" ('kn', 1.9, 0.952, 20, 1)]"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data = []\n",
"for lang_code in set(list(success_langs_vc.keys()) + list(error_langs_vc.keys())):\n",
" n_success = success_langs_vc.get(lang_code, 0)\n",
" n_error = error_langs_vc.get(lang_code, 0)\n",
" success_frac = n_success / (n_success + n_error)\n",
" lang_duration_h = round(np.sum(lang_durations.get(lang_code, [0])) / 60 / 60, 1)\n",
" data.append((lang_code, lang_duration_h, round(success_frac, 3), n_success, n_error))\n",
"sorted(data, key=lambda x: x[1], reverse=True)"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "b9f762bb",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"[('kn', 1.9, 0.952, 20, 1),\n",
" ('mr', 14.7, 0.933, 111, 8),\n",
" ('ko', 2888.8, 0.922, 21304, 1807),\n",
" ('zh-SG', 65.9, 0.922, 331, 28),\n",
" ('az', 19.6, 0.915, 195, 18),\n",
" ('bs', 28.3, 0.91, 162, 16),\n",
" ('uk', 517.7, 0.909, 3532, 354),\n",
" ('el', 570.5, 0.9, 3305, 369),\n",
" ('tr', 2015.7, 0.899, 11876, 1329),\n",
" ('ru', 6069.7, 0.898, 36553, 4149),\n",
" ('sr-Latn', 60.0, 0.892, 397, 48),\n",
" ('id', 2510.9, 0.89, 14385, 1771),\n",
" ('th', 609.6, 0.89, 3815, 471),\n",
" ('zh', 1140.5, 0.887, 6175, 783),\n",
" ('sk', 182.0, 0.884, 1836, 240),\n",
" ('vi', 974.9, 0.882, 6162, 823),\n",
" ('sr', 263.1, 0.869, 1463, 220),\n",
" ('et', 71.8, 0.867, 509, 78),\n",
" ('ja', 2090.5, 0.865, 14009, 2179),\n",
" ('hr', 169.0, 0.863, 1182, 187),\n",
" ('sv', 655.8, 0.86, 4387, 713),\n",
" ('kk', 47.3, 0.86, 416, 68),\n",
" ('ml', 18.1, 0.859, 158, 26),\n",
" ('lt', 119.8, 0.859, 917, 150),\n",
" ('zh-CN', 1152.0, 0.854, 5196, 887),\n",
" ('zh-Hans', 1697.7, 0.854, 8668, 1480),\n",
" ('ms', 124.8, 0.853, 763, 131),\n",
" ('hu', 378.9, 0.852, 2592, 452),\n",
" ('sr-Cyrl', 6.6, 0.851, 40, 7),\n",
" ('cs', 569.7, 0.851, 4671, 819),\n",
" ('sl', 99.4, 0.85, 881, 156),\n",
" ('en-AU', 2.8, 0.848, 78, 14),\n",
" ('no', 228.0, 0.846, 2195, 399),\n",
" ('ne', 7.2, 0.846, 88, 16),\n",
" ('de-CH', 44.0, 0.843, 371, 69),\n",
" ('fi', 551.8, 0.841, 3002, 567),\n",
" ('fil', 141.2, 0.84, 1218, 232),\n",
" ('bg', 179.5, 0.84, 1418, 271),\n",
" ('de-DE', 468.1, 0.835, 2885, 571),\n",
" ('sq', 42.7, 0.833, 504, 101),\n",
" ('zh-HK', 36.8, 0.822, 222, 48),\n",
" ('lv', 103.5, 0.819, 828, 183),\n",
" ('ro', 402.5, 0.817, 2454, 550),\n",
" ('pa', 6.0, 0.809, 38, 9),\n",
" ('si', 49.2, 0.799, 354, 89),\n",
" ('hi', 1119.7, 0.797, 8884, 2256),\n",
" ('da', 376.4, 0.788, 4723, 1269),\n",
" ('pl', 1923.2, 0.77, 10476, 3122),\n",
" ('de', 2628.1, 0.766, 18295, 5587),\n",
" ('fr-FR', 286.2, 0.75, 1750, 582),\n",
" ('fr-CA', 230.2, 0.749, 2260, 756),\n",
" ('de-AT', 28.5, 0.743, 263, 91),\n",
" ('nl-NL', 93.9, 0.741, 966, 337),\n",
" ('bn', 89.4, 0.738, 885, 314),\n",
" ('fr-CH', 8.9, 0.736, 92, 33),\n",
" ('te', 26.6, 0.732, 243, 89),\n",
" ('es-US', 69.8, 0.715, 561, 224),\n",
" ('en-GB', 7230.1, 0.705, 50977, 21368),\n",
" ('pt-BR', 1482.3, 0.699, 10953, 4713),\n",
" ('nl-BE', 32.9, 0.695, 376, 165),\n",
" ('es-ES', 433.8, 0.684, 3075, 1419),\n",
" ('en-US', 12520.0, 0.677, 76284, 36341),\n",
" ('en-CA', 780.0, 0.656, 5166, 2715),\n",
" ('tlh', 58.4, 0.656, 103, 54),\n",
" ('pt-PT', 113.5, 0.649, 930, 503),\n",
" ('ta', 113.3, 0.637, 796, 453),\n",
" ('es-419', 1309.8, 0.632, 10177, 5923),\n",
" ('nl', 690.4, 0.623, 7270, 4394),\n",
" ('en-IE', 99.8, 0.621, 686, 419),\n",
" ('fr', 2641.6, 0.611, 19657, 12507),\n",
" ('zh-TW', 124.2, 0.601, 1088, 721),\n",
" ('fr-BE', 14.5, 0.538, 134, 115),\n",
" ('hy', 33.9, 0.523, 384, 350),\n",
" ('en-IN', 859.9, 0.514, 5971, 5651),\n",
" ('en', 82282.8, 0.501, 539203, 537333),\n",
" ('it', 1019.7, 0.482, 7999, 8588),\n",
" ('pt', 1387.3, 0.464, 10342, 11946),\n",
" ('es', 3021.4, 0.442, 22061, 27906),\n",
" ('ca', 107.6, 0.383, 841, 1353),\n",
" ('es-MX', 231.5, 0.198, 1773, 7196)]"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sorted(data, key=lambda x: x[2], reverse=True)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "552e0c55",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"too few views 11\n",
"dtype: int64"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.Series(error_data[\"ky\"]).value_counts()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ea1374b0",
"metadata": {},
"outputs": [],
"source": [
"es, fr, de, it -- casing information not good\n",
"pt -- punctuation information not good"
]
},
{
"cell_type": "code",
"execution_count": 177,
"id": "9c549b80",
"metadata": {},
"outputs": [],
"source": [
"n = 0\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/failed_metas.jsonl\") as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if \"lang_code\" in m and m[\"lang_code\"].split(\"-\")[0] == \"pt\" and \"punctuation information\" in m[\"error_msg\"]:\n",
" if n == 4:\n",
" break\n",
" n += 1"
]
},
{
"cell_type": "code",
"execution_count": 178,
"id": "542e36af",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'success': False,\n",
" 'id': '6AgfAcAqV-8',\n",
" 'lang_code': 'pt',\n",
" 'vtt_fn': '6AgfAcAqV-8.pt.vtt',\n",
" 'audio_fn': '6AgfAcAqV-8.webm',\n",
" 'error_msg': \" -- punctuation information not good\"}"
]
},
"execution_count": 178,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"m"
]
},
{
"cell_type": "code",
"execution_count": 179,
"id": "60b9c2d6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WEBVTT\r\n",
"Kind: captions\r\n",
"Language: pt\r\n",
"\r\n",
"00:00:00.040 --> 00:00:06.100\r\n",
"Olá taurinos!! bem-vindos ao religare tarot \r\n",
"quântico e terapêutico eu sou Roberta e hoje \r\n",
"\r\n",
"00:00:06.100 --> 00:00:14.860\r\n",
"trago para você mais uma leitura semanal atemporal \r\n",
"na cruz celta e com o tema livre tá bom para você \r\n",
"\r\n",
"00:00:14.860 --> 00:00:21.790\r\n",
"que tá chegando por aqui pela primeiríssima vez \r\n",
"seja muito bem-vindo seja muito bem-vinda se você \r\n",
"\r\n",
"00:00:21.790 --> 00:00:27.550\r\n",
"gostasse você se identificar com canal Fique à \r\n",
"vontade para se inscrever ative as notificações \r\n",
"\r\n",
"00:00:27.550 --> 00:00:35.020\r\n",
"porque aí toda vez que lançaram conteúdo novo \r\n",
"você vai ser avisado ou avisada tá se você gostar \r\n",
"\r\n",
"00:00:35.020 --> 00:00:39.940\r\n",
"do vídeo fique à vontade para deixar seu like \r\n",
"fica à vontade também para Compartilhar esse \r\n",
"\r\n",
"00:00:39.940 --> 00:00:46.270\r\n",
"vídeo com os seus amigos taurinos se quiser \r\n",
"saber sobre leituras particulares curso de \r\n",
"\r\n",
"00:00:46.270 --> 00:00:52.900\r\n",
"tarot quiser conhecer minha loja online tudo isso \r\n",
"você encontra na descrição do vídeo lembro sempre \r\n",
"\r\n",
"00:00:52.900 --> 00:00:59.410\r\n",
"que as leituras aqui do canal são leituras \r\n",
"Gerais logo não leve para o lado pessoal tá \r\n",
"\r\n",
"00:01:00.280 --> 00:01:04.900\r\n",
"bom Então significa que nem sempre eu vou \r\n",
"ter uma leitura que vai se encaixar com a \r\n",
"\r\n",
"00:01:04.900 --> 00:01:09.640\r\n",
"sua realidade se você tá assistindo tá \r\n",
"percebendo que não bate nada contigo não \r\n",
"\r\n",
"00:01:09.640 --> 00:01:17.320\r\n",
"fique tentando encaixar a leitura por \r\n",
"força na sua vida tá bom vamos ver aqui\r\n",
"\r\n",
"00:01:23.070 --> 00:01:31.020\r\n",
"e apesar de hoje estar usando Neo tarô de \r\n",
"Oswald ti eu lembro sempre para vocês que \r\n",
"\r\n",
"00:01:31.020 --> 00:01:36.300\r\n",
"para interpretação de arcanos maiores eu vou \r\n",
"sempre me pautar pela tradição do tarot Não me \r\n",
"\r\n",
"00:01:36.300 --> 00:01:42.060\r\n",
"pauto pela pela pela linha indiana Muito menos \r\n",
"pela linha da Golden não tá vou sempre me faltar \r\n",
"\r\n",
"00:01:42.060 --> 00:01:50.550\r\n",
"pela tradição do Tarot independente do Deck que \r\n",
"eu uso e e é isso tá bom então vocês podem ser \r\n",
"\r\n",
"00:01:50.550 --> 00:01:55.680\r\n",
"que vocês associação entre arcanos maiores e \r\n",
"signos e Planetas seja um pouco diferente do \r\n",
"\r\n",
"00:01:55.680 --> 00:02:01.590\r\n",
"que vocês estão acostumados Mas isso é porque eu \r\n",
"sigo uma outra linha Tá bom vamos lá energia do \r\n",
"\r\n",
"00:02:01.590 --> 00:02:08.790\r\n",
"fundo do macho nós temos aqui um sete de copas \r\n",
"Olha cuidado porque sete de copas quando vibra \r\n",
"\r\n",
"00:02:08.790 --> 00:02:16.110\r\n",
"no no no sentido luz quando esse akane está \r\n",
"está vibrando no sentido luz ele fala de dar \r\n",
"\r\n",
"00:02:16.110 --> 00:02:22.330\r\n",
"importância de você ir dar um outro direcionamento \r\n",
"com um bom direcionamento para os seus sentimentos \r\n",
"\r\n",
"00:02:22.330 --> 00:02:29.890\r\n",
"é só pelo que eu estou entendendo aqui esse \r\n",
"sete de copas ele pode representar ilusões \r\n",
"\r\n",
"00:02:29.890 --> 00:02:37.870\r\n",
"projeções alguém que está mentindo para si mesmo \r\n",
"alguém está mentindo para si mesma alguém que \r\n",
"\r\n",
"00:02:37.870 --> 00:02:43.660\r\n",
"está vendo numa situação algo que realmente não \r\n",
"existe algo que não é exatamente da forma como \r\n",
"\r\n",
"00:02:43.660 --> 00:02:49.840\r\n",
"está vendo tá isso tanto pode ser para o bem \r\n",
"Como pode ser normal enfim aí vai de cada um \r\n",
"\r\n",
"00:02:49.840 --> 00:02:56.890\r\n",
"afinal de contas essa leitura para milhares de \r\n",
"pessoas Então tome cuidado com as ilusões e de \r\n",
"\r\n",
"00:02:56.890 --> 00:03:01.870\r\n",
"um bom direcionamento para as suas emoções para \r\n",
"os seus sentimentos não fique fazendo projeções \r\n",
"\r\n",
"00:03:01.870 --> 00:03:08.380\r\n",
"no outro ou numa determinada situação e eu acho \r\n",
"que no fundo no fundo todo você já sabe disso \r\n",
"\r\n",
"00:03:08.380 --> 00:03:15.820\r\n",
"é porque ó você no seu sentimento mais sincero \r\n",
"profundo e verdadeiro dois de paus aqui é você \r\n",
"\r\n",
"00:03:15.820 --> 00:03:22.340\r\n",
"literalmente parado feito um dois de paus porque \r\n",
"tem um impasse Zinho aí dentro de você você tá E \r\n",
"\r\n",
"00:03:22.340 --> 00:03:28.220\r\n",
"aí já seus próximos passos mas você tá me travado \r\n",
"você não tá conseguindo agir porque é isso uma \r\n",
"\r\n",
"00:03:28.220 --> 00:03:36.410\r\n",
"energia que te Ronda que te circunda aí ó o Arcano \r\n",
"maior o enforcado aí o sacrifício ai não sei o quê \r\n",
"\r\n",
"00:03:36.410 --> 00:03:46.460\r\n",
"aí Tá tudo está enganado calma aqui é o sacro \r\n",
"Ofício é o Ofício sagrado da observação tá aqui \r\n",
"\r\n",
"00:03:46.460 --> 00:03:54.380\r\n",
"é você ó olhando para o seu passado tentando \r\n",
"projetar o seu futuro por quê Porque o Arcano \r\n",
"\r\n",
"00:03:54.380 --> 00:04:01.160\r\n",
"quatro enforcado é um ar canto comprometimento \r\n",
"a pessoa tá comprometida Ela tá presa pelo pé \r\n",
"\r\n",
"00:04:02.090 --> 00:04:09.350\r\n",
"PS4 aqui que ele faz com a perna ele querendo \r\n",
"firmar na matéria tá ele querendo te comprometer \r\n",
"\r\n",
"00:04:09.350 --> 00:04:14.510\r\n",
"e queiram se firmar na matéria mas antes ele \r\n",
"precisa parar ele precisa pensar ele precisa \r\n",
"\r\n",
"00:04:14.510 --> 00:04:21.140\r\n",
"observar ele precisa enxergar essa situação por \r\n",
"todos os ângulos por todas as perspectivas e não \r\n",
"\r\n",
"00:04:21.140 --> 00:04:26.600\r\n",
"por isso que tem esse impasse e Zinho aqui né Você \r\n",
"tá de meio travado você quer fazer alguma você não \r\n",
"\r\n",
"00:04:26.600 --> 00:04:31.610\r\n",
"sabe assim você não tá Literalmente com freio de \r\n",
"mão puxado por quê Porque ele andou toma tanta \r\n",
"\r\n",
"00:04:31.610 --> 00:04:37.700\r\n",
"bordoada na vida né tudo que agora fala Opa Mas \r\n",
"por que isso eu tô perguntando porque isso gente \r\n",
"\r\n",
"00:04:37.700 --> 00:04:46.220\r\n",
"Ah tá porque porque que touro tem mente valete \r\n",
"de copas e esse valete de copas aqui não tem a \r\n",
"\r\n",
"00:04:46.220 --> 00:04:52.580\r\n",
"ver com o pedido de desculpas é que é literalmente \r\n",
"você querendo Abrir sua boquinha e falar dos seus \r\n",
"\r\n",
"00:04:52.580 --> 00:05:00.110\r\n",
"sentimentos para um ser de luz falar do que você \r\n",
"sente por essa pessoa de toda a preço do que você \r\n",
"\r\n",
"00:05:00.110 --> 00:05:07.710\r\n",
"pretende Pererê Parará é mais uma energia que \r\n",
"te segura neste momento rei de ouros graças a \r\n",
"\r\n",
"00:05:07.710 --> 00:05:15.420\r\n",
"meu bom Deus né aqui um pouquinho de razão aqui \r\n",
"é você tendo um pouco mais de cautela esse Rei \r\n",
"\r\n",
"00:05:15.420 --> 00:05:21.210\r\n",
"de ouros aqui é literalmente você querendo ter o \r\n",
"controle porque se regionais aqui como ele tá te \r\n",
"\r\n",
"00:05:21.210 --> 00:05:30.060\r\n",
"desafiando aqui energia sombria dele ele tá assim \r\n",
"ele tá segurando não calma Segura a Onda pensa um \r\n",
"\r\n",
"00:05:30.060 --> 00:05:37.980\r\n",
"pouquinho mais em você não deu não deu não seja \r\n",
"precipitado não seja precipitada né observa a \r\n",
"\r\n",
"00:05:37.980 --> 00:05:43.020\r\n",
"melhor esse terreno está firme tá seguro ou \r\n",
"seja o lado sombrio do Rei de Ouro está te \r\n",
"\r\n",
"00:05:43.020 --> 00:05:48.780\r\n",
"ajudando porque ele tá querendo o teu controle o \r\n",
"auto-controle aquela pessoa que quer ganhar sempre \r\n",
"\r\n",
"00:05:48.780 --> 00:05:53.910\r\n",
"que quer levar à vontade Então nesse momento \r\n",
"esse teu lado sombrio tá te ajudando porque \r\n",
"\r\n",
"00:05:53.910 --> 00:06:02.080\r\n",
"ele tá ele tá segurando a sua onda toma cuidado \r\n",
"com as ilusões vai a base para a estruturação \r\n",
"\r\n",
"00:06:02.080 --> 00:06:09.850\r\n",
"a situação aqui Cavaleiro de copas Cavaleiro de \r\n",
"copas aqui é você talvez achando que tem ali uma \r\n",
"\r\n",
"00:06:09.850 --> 00:06:17.350\r\n",
"oportunidade de um novo amor de uma nova amizade \r\n",
"né ou se esse Cavaleiro de copas tiver vibrando \r\n",
"\r\n",
"00:06:17.350 --> 00:06:25.960\r\n",
"no sentido Sombrio e representar uma pessoa aqui \r\n",
"pode ser aquela pessoa que está ali Fazendo um \r\n",
"\r\n",
"00:06:25.960 --> 00:06:32.920\r\n",
"charme aquele Don Juan naquela Maria Ruan né Ou \r\n",
"seja você a pessoa time jogando um charme mas não \r\n",
"\r\n",
"00:06:32.920 --> 00:06:39.880\r\n",
"tinha nada de sério com você só tá ali mesmo para \r\n",
"jogar um charminho para fazer uma graça né para \r\n",
"\r\n",
"00:06:39.880 --> 00:06:44.320\r\n",
"fazer exposição da figura na mediana mas querer \r\n",
"algo sério mesmo não quer porque o Cavaleiro é \r\n",
"\r\n",
"00:06:44.320 --> 00:06:49.360\r\n",
"que tá sempre passagem na vida das pessoas ele tá \r\n",
"ali para ficar Bandeira ele não está ali para se \r\n",
"\r\n",
"00:06:49.360 --> 00:06:55.390\r\n",
"comprometer com ninguém uma energia que vem se \r\n",
"afastando de você com esse dez de espadas aqui \r\n",
"\r\n",
"00:06:55.390 --> 00:07:02.090\r\n",
"esse dez de espadas pode representar você no \r\n",
"luto eterno aí por conta de um fechamento de \r\n",
"\r\n",
"00:07:02.090 --> 00:07:08.390\r\n",
"o que não acaba nunca e você sofrendo por aquilo \r\n",
"aquela coisa então agora você resolveu fechar \r\n",
"\r\n",
"00:07:08.390 --> 00:07:13.850\r\n",
"esse ciclo e dá sequência na sua vida aí aparece \r\n",
"esse bonito aparece essa bonita aí você fica aí \r\n",
"\r\n",
"00:07:13.850 --> 00:07:20.330\r\n",
"ó ai que será que fala será que não falo Será \r\n",
"que isso se ela que aquilo uma energia que vem \r\n",
"\r\n",
"00:07:20.330 --> 00:07:27.140\r\n",
"para você a curto prazo valete de espadas aqui \r\n",
"medo ai que medo senhor vamos lá esse valete de \r\n",
"\r\n",
"00:07:27.140 --> 00:07:34.100\r\n",
"espadas aqui me fala assim de uma conversa ou \r\n",
"ainda você stalkeando o ser de luz o servilost \r\n",
"\r\n",
"00:07:34.100 --> 00:07:39.500\r\n",
"stalkeando tal e coisa e coisa e tal mas muita \r\n",
"conversa principalmente conversa eletrônica né \r\n",
"\r\n",
"00:07:39.500 --> 00:07:47.060\r\n",
"ligação ou troca de mensagem né Mari pelo Zap né \r\n",
"passarinho que previsível para lave frio para cá \r\n",
"\r\n",
"00:07:47.060 --> 00:07:52.940\r\n",
"e aquela coisa toda né bom como é que você está se \r\n",
"comportando na energia no Arcano maior o eremita \r\n",
"\r\n",
"00:07:52.940 --> 00:07:58.880\r\n",
"tá tentando ter sabedoria né touro pelo menos \r\n",
"isso é uma tá tentando né atentando ele ser uma \r\n",
"\r\n",
"00:07:58.880 --> 00:08:06.930\r\n",
"pessoa sabe A Misteriosa tem hora que você Oi você \r\n",
"some de Ceni some muita coisa né como é que o ser \r\n",
"\r\n",
"00:08:06.930 --> 00:08:14.400\r\n",
"de luz está te vendo na energia do Arcano maior \r\n",
"Pois é namorados a pessoa tá será que touro tem \r\n",
"\r\n",
"00:08:14.400 --> 00:08:21.090\r\n",
"outra pessoa Será que todo tem alguém Será \r\n",
"que tolo precisa escolher e imprimir outra \r\n",
"\r\n",
"00:08:21.090 --> 00:08:26.580\r\n",
"pessoa né pessoal trade com aquela mente assim \r\n",
"sabe muito criativa por isso talvez as telhas \r\n",
"\r\n",
"00:08:26.580 --> 00:08:33.630\r\n",
"stalkeando para que ele entender Por que que você \r\n",
"sumiu né Por que que você sumiu o que você quer o \r\n",
"\r\n",
"00:08:33.630 --> 00:08:39.960\r\n",
"que você deseja o três de copas a manifestação \r\n",
"do amor né você quer viver essa experiência mas \r\n",
"\r\n",
"00:08:39.960 --> 00:08:47.070\r\n",
"você tem nesse cinco de espadas aqui ó você tem \r\n",
"este se arrebentar nesses Boo Degar se lascar \r\n",
"\r\n",
"00:08:47.070 --> 00:08:53.970\r\n",
"todos eu acho que a toda e aí que o resultado \r\n",
"oito de paus Ai ai esse oito de pausa aqui ó \r\n",
"\r\n",
"00:08:54.950 --> 00:09:00.710\r\n",
"e esse oito de boxe tanto pode me falar de você \r\n",
"literalmente ou falar para oito de paus pode \r\n",
"\r\n",
"00:09:00.710 --> 00:09:08.090\r\n",
"representar declaração oito de paus pode falar \r\n",
"de uma de um deslocamento curto mas também de uma \r\n",
"\r\n",
"00:09:08.090 --> 00:09:14.270\r\n",
"certa precipitação tá então para comer uma leitura \r\n",
"para milhares de pessoas em vai passar quer saber \r\n",
"\r\n",
"00:09:14.270 --> 00:09:18.740\r\n",
"eu vou fazer mesmo e essa Roberta maluca não \r\n",
"sabe que tá falando eu vou lá e vou fazer aí \r\n",
"\r\n",
"00:09:18.740 --> 00:09:23.510\r\n",
"faço é um direito seu livre-arbítrio não pode \r\n",
"fazer para você não fazer nada é eu nunca digo \r\n",
"\r\n",
"00:09:23.510 --> 00:09:28.550\r\n",
"para o consulente que ele tem que fazer né que \r\n",
"isso é manipulação a vou dar um conselho então do \r\n",
"\r\n",
"00:09:28.550 --> 00:09:34.610\r\n",
"Conselho nenhum mostra energia a decisão é sempre \r\n",
"sua de acordo com o seu entendimento de acordo com \r\n",
"\r\n",
"00:09:34.610 --> 00:09:41.180\r\n",
"a sua vontade de acordo com seu livre arbítrio aí \r\n",
"beleza você fala toma decisão ou você se deslocou \r\n",
"\r\n",
"00:09:41.180 --> 00:09:46.400\r\n",
"você falou ou você agiu enfim Seja lá o que \r\n",
"for perguntei beleza e quais os desdobramentos \r\n",
"\r\n",
"00:09:47.460 --> 00:09:59.710\r\n",
"e Nove de copas e a lua e Aqui nós temos o Arcano \r\n",
"maior o perdão o rei de o rei de espadas Arcano \r\n",
"\r\n",
"00:09:59.710 --> 00:10:06.490\r\n",
"maior nenhum Tá bom então aqui quais são os signos \r\n",
"que eu tô vendo aqui eu tô vendo libra eu tô vendo \r\n",
"\r\n",
"00:10:06.490 --> 00:10:13.210\r\n",
"aquário eu tô vendo touro e eu estou vendo Leão \r\n",
"tá então São esses os prováveis signos do ser \r\n",
"\r\n",
"00:10:13.210 --> 00:10:20.260\r\n",
"de luz então aqui se nove de copas e você vai \r\n",
"lá age agiu coisa e tal e nossa nove de copas \r\n",
"\r\n",
"00:10:20.260 --> 00:10:28.600\r\n",
"você me sentindo Radiante feliz da vida só que \r\n",
"logo na sequência nós temos o que o Arcano maior \r\n",
"\r\n",
"00:10:28.600 --> 00:10:37.510\r\n",
"a lua medos dúvidas e inseguranças alguém que vai \r\n",
"precisar sair da sua zona de conforto alguém que \r\n",
"\r\n",
"00:10:37.510 --> 00:10:45.040\r\n",
"vai precisar agir diferente e esse Rei de espadas \r\n",
"aqui trazendo a Lucidez que tirando dessas ilusões \r\n",
"\r\n",
"00:10:45.040 --> 00:10:50.660\r\n",
"e te trazendo para real tá que o rei de espadas \r\n",
"ele é o rei que ele enxerga a vida como é Ah e \r\n",
"\r\n",
"00:10:50.660 --> 00:10:56.300\r\n",
"não com óculos cor-de-rosa achando que tudo é \r\n",
"lindo tudo é maravilhoso tá então e se repita \r\n",
"\r\n",
"00:10:56.300 --> 00:11:02.630\r\n",
"por mais que você ao todo iludido vai lá faz \r\n",
"alguma coisa acontece será alguma situação é \r\n",
"\r\n",
"00:11:02.630 --> 00:11:07.040\r\n",
"que vai te deixar todo feliz e Radiante logo na \r\n",
"sequência começa a bater dúvida e insegurança \r\n",
"\r\n",
"00:11:07.040 --> 00:11:12.650\r\n",
"e Incerteza mais o rede espada está ali ó para \r\n",
"te trazer para real para ele para te mostrar o \r\n",
"\r\n",
"00:11:12.650 --> 00:11:21.380\r\n",
"que você não tá querendo ver tá pode ser também \r\n",
"que você Fiquei todo feliz e Saltitante e o ser \r\n",
"\r\n",
"00:11:21.380 --> 00:11:29.900\r\n",
"de luz por alguma razão tres Easy se desafiar um \r\n",
"pouco mais sair da zona de conforto dele ou dela \r\n",
"\r\n",
"00:11:29.900 --> 00:11:35.600\r\n",
"e pode ser que essa pessoa não esteja disposta a \r\n",
"isso no momento Então essa pessoa tendência dela \r\n",
"\r\n",
"00:11:35.600 --> 00:11:41.840\r\n",
"ficar mais fria mais distante começar a te tratar \r\n",
"com uma certa indiferença Então tome cuidado para \r\n",
"\r\n",
"00:11:41.840 --> 00:11:48.500\r\n",
"não fazer projeções cuidado Mas eles longe de um \r\n",
"outro direcionamento para os seus sentimentos tô \r\n",
"\r\n",
"00:11:48.500 --> 00:11:54.510\r\n",
"falando que é para você desistir da pessoa E \r\n",
"é só para você observar o que é realidade e o \r\n",
"\r\n",
"00:11:54.510 --> 00:12:00.420\r\n",
"que é projeção e o que a fantasia da sua mente \r\n",
"do seu coração nesse contexto tá vamos lá vamos \r\n",
"\r\n",
"00:12:00.420 --> 00:12:06.750\r\n",
"para a próxima a leitura Mas e aí tu fica tudo de \r\n",
"boa com você tá tudo tranquilo tá tudo na santa \r\n",
"\r\n",
"00:12:06.750 --> 00:12:13.440\r\n",
"paz do Senhor Espero que sim tá me conta aí De \r\n",
"onde você me assistir Tá frio aí gente aqui no \r\n",
"\r\n",
"00:12:13.440 --> 00:12:17.250\r\n",
"Rio de Janeiro tá bagunça tem dia que tá frito em \r\n",
"dia que tá calor Tem dia que passa o dia inteiro \r\n",
"\r\n",
"00:12:17.250 --> 00:12:23.850\r\n",
"uma friaca horrorosa e das 6 horas da tarde \r\n",
"a temperatura muda fica quentinho a esfria de \r\n",
"\r\n",
"00:12:23.850 --> 00:12:32.250\r\n",
"novo de madrugada e aí amanhece o dia no gelo no \r\n",
"gelo insuportável e melhora um pouquinho hoje por \r\n",
"\r\n",
"00:12:32.250 --> 00:12:40.960\r\n",
"exemplo amanheceu o dia frio aí já tirei o casal \r\n",
"que eu já tô sem casaco Bom vamos lá é essa daqui \r\n",
"\r\n",
"00:12:40.960 --> 00:12:46.750\r\n",
"é para você que tem de fato um ser uma link tem \r\n",
"que falar com um ser de luz tá vamos ver que o \r\n",
"\r\n",
"00:12:46.750 --> 00:12:56.140\r\n",
"seguinte passado recente o momento atual futuro \r\n",
"próximo dessa relação uma energia desafiadora você \r\n",
"\r\n",
"00:12:56.140 --> 00:13:03.550\r\n",
"em relação aos indivíduos ser de luz em relação \r\n",
"a você o caminho e o desfecho como ver aqui ó\r\n",
"\r\n",
"00:13:06.380 --> 00:13:11.180\r\n",
"um daqueles que quiserem saber sobre \r\n",
"leituras particulares informações \r\n",
"\r\n",
"00:13:11.180 --> 00:13:14.570\r\n",
"na descrição do vídeo na descrição do \r\n",
"vídeo você encontra o link para minha \r\n",
"\r\n",
"00:13:14.570 --> 00:13:18.640\r\n",
"loja online onde você vai encontrar esse \r\n",
"carro aqui tá mais bonito Dourado tá e\r\n",
"\r\n",
"00:13:18.640 --> 00:13:26.650\r\n",
"aqueles que fizeram deixar um laço \r\n",
"com o like no vídeo eu agradeço\r\n",
"\r\n",
"00:13:29.770 --> 00:13:34.210\r\n",
"Oi de novo a lua gente ó de novo a lua\r\n",
"\r\n",
"00:13:36.920 --> 00:13:44.540\r\n",
"I am Oi de novo valete de copas e valete \r\n",
"de espadas energia do fundo do Márcio \r\n",
"\r\n",
"00:13:44.540 --> 00:13:53.510\r\n",
"cinco de ouros um alguém se sentindo \r\n",
"rejeitado alguém se sentindo rejeitada \r\n",
"\r\n",
"00:13:54.490 --> 00:14:02.650\r\n",
"Oi alguém aí numa carência danada hein fininho \r\n",
"passado recente aqui com o Arcano maior a Lua ou \r\n",
"\r\n",
"00:14:02.650 --> 00:14:10.480\r\n",
"perdão a sacerdotisa serviço é regido pela lua \r\n",
"né e assim aqui me fala de uma revelação algo \r\n",
"\r\n",
"00:14:10.480 --> 00:14:17.170\r\n",
"foi revelado algo veio à tona nessa relação ao \r\n",
"passado recente no momento presente como Arcano \r\n",
"\r\n",
"00:14:17.170 --> 00:14:26.170\r\n",
"maior a força aqui me mostra de alguém tomando o \r\n",
"controle ou retomando o controle dessa situação me \r\n",
"\r\n",
"00:14:26.170 --> 00:14:31.420\r\n",
"fala também de uma pessoa com auto controle \r\n",
"pessoal se segurando aqui né para não meter \r\n",
"\r\n",
"00:14:31.420 --> 00:14:38.740\r\n",
"os pés pelas mãos na situação mas não futuro \r\n",
"próximo aqui com o Arcano maior um mundo aqui \r\n",
"\r\n",
"00:14:38.740 --> 00:14:45.220\r\n",
"me mostra o seguinte aqui me mostra aqui para \r\n",
"que vocês vocês eu digo as duas partes dessa \r\n",
"\r\n",
"00:14:45.220 --> 00:14:54.500\r\n",
"relação tenham êxito aqui me fala da importância \r\n",
"de vocês primeiro equilibrar em razão com emoção \r\n",
"\r\n",
"00:14:54.500 --> 00:15:01.850\r\n",
"e as ações ações bem estruturadas as paixões \r\n",
"do impulso entusiasmo tá aqui me mostra também \r\n",
"\r\n",
"00:15:01.850 --> 00:15:09.650\r\n",
"vocês que o entendimento de que vocês são as \r\n",
"pessoas certas no lugar certo na hora certa com \r\n",
"\r\n",
"00:15:09.650 --> 00:15:15.860\r\n",
"quem vocês tem que estar para que para que haja \r\n",
"um amadurecimento uma energia desafiadora aqui \r\n",
"\r\n",
"00:15:15.860 --> 00:15:22.550\r\n",
"nessa questão foram fechamento de um ciclo muito \r\n",
"doloroso foi uma briga e algo que representou uma \r\n",
"\r\n",
"00:15:22.550 --> 00:15:28.970\r\n",
"ruptura muito dolorosa tem uma pessoa que fica \r\n",
"fazendo manutenção dessa tristeza dessa dor \r\n",
"\r\n",
"00:15:29.920 --> 00:15:38.020\r\n",
"e até mesmo assim como a energia de vitimização né \r\n",
"Toda hora Fica remoendo remoendo remoendo como é \r\n",
"\r\n",
"00:15:38.020 --> 00:15:45.040\r\n",
"que você é touro está em relação ao ser de longe \r\n",
"nove de ouros Você tá muito bem obrigado Você tá \r\n",
"\r\n",
"00:15:45.040 --> 00:15:50.080\r\n",
"com a sua energia aí dentro com a sua autoestima \r\n",
"muito bem trabalhada você não tá mais vibrando \r\n",
"\r\n",
"00:15:50.080 --> 00:15:57.040\r\n",
"nessa dependência do outro Como está o outro em \r\n",
"relação a você na energia do Arcano maior a lua \r\n",
"\r\n",
"00:15:57.040 --> 00:16:04.180\r\n",
"a que fala de dúvidas de inseguranças aqui me fala \r\n",
"de projeções aqui me fala de uma pessoa que nesse \r\n",
"\r\n",
"00:16:04.180 --> 00:16:09.760\r\n",
"momento que algo bacana que algo grandioso mas \r\n",
"não quer sair da zona de conforto e assim essa \r\n",
"\r\n",
"00:16:09.760 --> 00:16:14.590\r\n",
"pessoa precisa entender que para ter um resultado \r\n",
"diferente na vida dele ou dela ele precisa de uma \r\n",
"\r\n",
"00:16:14.590 --> 00:16:24.930\r\n",
"forma diferente né e qual caminho valete de copas \r\n",
"vejo aqui que alguém vai tentar uma conversa de \r\n",
"\r\n",
"00:16:24.930 --> 00:16:30.420\r\n",
"coração para coração abrir o Coração Falar dos \r\n",
"sentimentos e talvez até um pedido de desculpas \r\n",
"\r\n",
"00:16:30.420 --> 00:16:37.650\r\n",
"mas o resultado um valete de espadas é que eu vejo \r\n",
"que essa conversa ela não pode ser tão amistosa \r\n",
"\r\n",
"00:16:37.650 --> 00:16:42.810\r\n",
"assim o vejo que é que alguém pode falar de uma \r\n",
"maneira um pouco mais de Leviana não ter o certo \r\n",
"\r\n",
"00:16:42.810 --> 00:16:48.750\r\n",
"Cuidado um certo trato com as palavras e alguém \r\n",
"sair ferido e machucado e se sentindo assim ó \r\n",
"\r\n",
"00:16:48.750 --> 00:16:57.780\r\n",
"rejeitado ou rejeitada Tá certo complicado aqui \r\n",
"né touro deu ruim aqui né depois ele me contem \r\n",
"\r\n",
"00:16:57.780 --> 00:17:02.400\r\n",
"nos comentários o que foi que aconteceu de fato \r\n",
"aí porque que tá esse símbolo de chamadas aqui \r\n",
"\r\n",
"00:17:02.400 --> 00:17:11.130\r\n",
"bom agora essa daqui também como você tem um \r\n",
"ser humaninho de fato na sua vida ou você tá \r\n",
"\r\n",
"00:17:11.130 --> 00:17:16.780\r\n",
"paquerando o ser humaninho você Tá paquerando ser \r\n",
"de luz né não sabe que a pessoa empolga na cognis \r\n",
"\r\n",
"00:17:16.780 --> 00:17:22.420\r\n",
"a vida não é um enigma a ser decifrado E então a \r\n",
"gente vai tentar decifrar o seu maninho aqui tá \r\n",
"\r\n",
"00:17:22.420 --> 00:17:28.720\r\n",
"Aqui nós temos o seguinte Quais são os pensamentos \r\n",
"dessa pessoa em relação a você quais os \r\n",
"\r\n",
"00:17:28.720 --> 00:17:35.470\r\n",
"sentimentos o que essa pessoa intenciona com você \r\n",
"qual é a tendência de atitude da pessoa contigo\r\n",
"\r\n",
"00:17:39.670 --> 00:17:39.730\r\n",
"E aí\r\n",
"\r\n",
"00:17:42.490 --> 00:17:47.620\r\n",
"e vamos ver como é que tá o \r\n",
"seu paninho com a vossa pessoa\r\n",
"\r\n",
"00:17:52.010 --> 00:17:54.230\r\n",
"e quais os pensamentos\r\n",
"\r\n",
"00:17:56.480 --> 00:17:58.220\r\n",
"G1\r\n",
"\r\n",
"00:18:03.710 --> 00:18:05.930\r\n",
"e quais os sentimentos\r\n",
"\r\n",
"00:18:05.930 --> 00:18:18.440\r\n",
"agonia só né gente hum hum acho \r\n",
"que a pessoa intenciona a E aí\r\n",
"\r\n",
"00:18:20.420 --> 00:18:25.500\r\n",
"E aí e qual é a tendência de atitudes\r\n",
"\r\n",
"00:18:27.790 --> 00:18:35.410\r\n",
"a energia do fundo do Março três de copas \r\n",
"a reconciliação aqui tem uma energia de \r\n",
"\r\n",
"00:18:35.410 --> 00:18:41.950\r\n",
"reconciliação três de copas 8 de ouros enfim aqui \r\n",
"tem uma energia de reconciliação aqui é o seguinte \r\n",
"\r\n",
"00:18:41.950 --> 00:18:47.050\r\n",
"como é que tá os pensamentos dessa pessoa em \r\n",
"relação a você essa pessoa sente que ela precisa \r\n",
"\r\n",
"00:18:47.050 --> 00:18:55.660\r\n",
"tomar atitude ela precisa ir à luta tá ela precisa \r\n",
"ter uma ter uma ideia um Insight de como ela vai \r\n",
"\r\n",
"00:18:55.660 --> 00:19:00.250\r\n",
"vai fazer para poder chegar até você Quais são \r\n",
"as palavras certeiras que ela precisa usar com \r\n",
"\r\n",
"00:19:00.250 --> 00:19:07.870\r\n",
"você tá é uma pessoa que assim ela é muito devagar \r\n",
"quase parando né ela tá ali numa rezei era coisa e \r\n",
"\r\n",
"00:19:07.870 --> 00:19:13.300\r\n",
"tal e fingir Mas eu vejo que essa pessoa ela fala \r\n",
"assim não do jeito que tá até agora Do jeito que \r\n",
"\r\n",
"00:19:13.300 --> 00:19:18.400\r\n",
"eu achei até hoje não funcionou preciso agir de \r\n",
"uma forma diferente então assim se eu sou uma \r\n",
"\r\n",
"00:19:18.400 --> 00:19:23.230\r\n",
"pessoa muito seca nas palavras se eu sou uma \r\n",
"pessoa muito dura muito frio eu preciso mudar \r\n",
"\r\n",
"00:19:23.230 --> 00:19:29.060\r\n",
"eu preciso transformar isso eu preciso ter uma \r\n",
"comunicação mais amorosa uma pô e mais amorosa \r\n",
"\r\n",
"00:19:29.060 --> 00:19:34.280\r\n",
"mais acolhedora senão touro não vai me aceitar \r\n",
"de volta eu dou tem que ser mais fofo tem que \r\n",
"\r\n",
"00:19:34.280 --> 00:19:40.880\r\n",
"ser mais fofa né porque emocionalmente aqui um \r\n",
"oito de espadas é que eu tô vendo uma angústia \r\n",
"\r\n",
"00:19:40.880 --> 00:19:46.820\r\n",
"uma sensação de culpa aquela pessoa que por mais \r\n",
"que tenha tentado se desvincular se desvencilhar \r\n",
"\r\n",
"00:19:46.820 --> 00:19:54.350\r\n",
"de você essa pessoa não consegue aqui essa lua que \r\n",
"me fala mesmo de de angústia de agonia a justiça \r\n",
"\r\n",
"00:19:54.350 --> 00:20:01.370\r\n",
"mas o louco seja essa pessoa ela tá angustiada \r\n",
"1° e emocionalmente em segura em estável sabe \r\n",
"\r\n",
"00:20:01.370 --> 00:20:07.190\r\n",
"enfim eu vejo que ela quer tomar uma decisão mas \r\n",
"estamos numa agonia só esse louco aqui também nos \r\n",
"\r\n",
"00:20:07.190 --> 00:20:13.910\r\n",
"sentimentos que me fala de uma pessoa imprevisível \r\n",
"assim emocionalmente essa pessoa é pura angústia \r\n",
"\r\n",
"00:20:13.910 --> 00:20:21.590\r\n",
"pura instabilidade pura insegurança nem a pessoa \r\n",
"se entendi direito né mas assim o que que essa \r\n",
"\r\n",
"00:20:21.590 --> 00:20:29.700\r\n",
"pessoa intenciona literalmente tomar uma atitude \r\n",
"a que o mago com as de paus para quê a reconstruir \r\n",
"\r\n",
"00:20:29.700 --> 00:20:37.050\r\n",
"essa relação em refazer essa relação reconstruir \r\n",
"essa história retomar o controle ter autocontrole \r\n",
"\r\n",
"00:20:37.050 --> 00:20:42.120\r\n",
"que essa pessoa pode ter feito uma bobagem aqui \r\n",
"então aqui várias só a gente sinceramente só que \r\n",
"\r\n",
"00:20:42.120 --> 00:20:49.440\r\n",
"eu tô vendo aqui isso aqui tanto pode ser um \r\n",
"relacionamento amoroso com uma amizade tá eu \r\n",
"\r\n",
"00:20:49.440 --> 00:20:55.350\r\n",
"tô vendo aqui que não é só uma relação amorosa \r\n",
"por isso aqui também pode ser uma amizade tá E \r\n",
"\r\n",
"00:20:55.350 --> 00:21:00.420\r\n",
"essa pessoa ela tá muito angustiada com a postura \r\n",
"que ela teve com atitude que ela teve ela sente \r\n",
"\r\n",
"00:21:00.420 --> 00:21:07.710\r\n",
"que ela precisa mudar tá tem aqui muita assim \r\n",
"muito sentimento de culpa de arrependimento de \r\n",
"\r\n",
"00:21:07.710 --> 00:21:16.020\r\n",
"angústia de dúvida de segurança de incerteza \r\n",
"de imprevisibilidade sabe é muito confuso que \r\n",
"\r\n",
"00:21:16.020 --> 00:21:20.820\r\n",
"essa pessoa está sentindo que ela tá assim ela \r\n",
"tá pensando mas ela tá emocionalmente confusa \r\n",
"\r\n",
"00:21:20.820 --> 00:21:27.790\r\n",
"e qual é a tendência de atitude dessa pessoa \r\n",
"que com sol a sacerdotisa o 4 de paus o 7 DP \r\n",
"\r\n",
"00:21:27.790 --> 00:21:35.860\r\n",
"Oi aqui é literalmente a pessoa tomando a coragem \r\n",
"saiu daí do casulinho dela enfrentando os medos \r\n",
"\r\n",
"00:21:35.860 --> 00:21:42.460\r\n",
"enfrentando é o teu ego o próprio ego dela para \r\n",
"tentar essa reconciliação vídeo essa pessoa se \r\n",
"\r\n",
"00:21:42.460 --> 00:21:48.220\r\n",
"movimentando tomando uma atitude revelando coisas \r\n",
"para você falando coisas para você que ela nunca \r\n",
"\r\n",
"00:21:48.220 --> 00:21:53.260\r\n",
"teve coragem de falar até então tá tendo uma \r\n",
"postura diferente mas parece ela vai ter que \r\n",
"\r\n",
"00:21:53.260 --> 00:21:58.750\r\n",
"enfrentar inclusive o próprio medo próprio \r\n",
"ego a própria insegurança isso aqui não é a \r\n",
"\r\n",
"00:21:58.750 --> 00:22:03.850\r\n",
"pessoa na defensiva Mas Ela É enfrentando \r\n",
"os medos dela enfrentando as inseguranças \r\n",
"\r\n",
"00:22:03.850 --> 00:22:09.790\r\n",
"dela para que essa reconciliação para que se \r\n",
"reencontra aconteça uma pessoa que sente muito \r\n",
"\r\n",
"00:22:09.790 --> 00:22:16.420\r\n",
"a sua falta uma pessoa que se arrepende \r\n",
"bastante do que fez se arrepende bastante \r\n",
"\r\n",
"00:22:16.420 --> 00:22:22.390\r\n",
"desse afastamento entre vocês tá bom touro é \r\n",
"isso espero de verdade poder ter te ajudado \r\n",
"\r\n",
"00:22:22.390 --> 00:22:28.580\r\n",
"com essa leitura beijo enorme no seu coração \r\n",
"fique com Deus uma ótima semana E até a E aí\r\n",
"\r\n"
]
}
],
"source": [
"!cat /data2/suno/data/harvest/youtube/audio/6AgfAcAqV-8.pt.vtt"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e520a6a5",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "84fdc827",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "20d31417",
"metadata": {},
"source": [
"### segmenting"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "641a7bc0",
"metadata": {},
"outputs": [],
"source": [
"import uuid\n",
"from suno_utils.harvest.youtube.constants_languages import NON_WHITESPACE_LANGS"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "1d66f257",
"metadata": {},
"outputs": [],
"source": [
"MIN_SEGMENT_DURATION_S = 2.0\n",
"MAX_SEGMENT_DURATION_S = 20.0\n",
"MAX_SILENCE_DURATION_S = 5.0\n",
"\n",
"\n",
"def create_segments(\n",
" annotated_lines,\n",
" lang_code,\n",
" min_segment_duration_s=MIN_SEGMENT_DURATION_S,\n",
" max_segment_duration_s=MAX_SEGMENT_DURATION_S,\n",
" max_silence_duration_s=MAX_SILENCE_DURATION_S\n",
"):\n",
" if len(annotated_lines) == 0:\n",
" return []\n",
" merge_char = \"\" if lang_code in NON_WHITESPACE_LANGS else \" \"\n",
" segments_info = []\n",
" for (start_s, end_s), text in annotated_lines:\n",
" if (\n",
" len(segments_info) > 0 and \n",
" end_s - segments_info[-1][0][0] <= max_segment_duration_s and\n",
" start_s - segments_info[-1][0][1] <= max_silence_duration_s\n",
" ):\n",
" segments_info[-1] = (\n",
" (segments_info[-1][0][0], end_s), \n",
" segments_info[-1][1] + merge_char + text\n",
" )\n",
" else:\n",
" segments_info.append(((start_s, end_s), text))\n",
" segments_info = [\n",
" ((start_s, end_s), text.strip(\"| \"), str(uuid.uuid4())) \n",
" for (start_s, end_s), text in segments_info\n",
" if end_s - start_s <= max_segment_duration_s and end_s - start_s >= min_segment_duration_s\n",
" ]\n",
" return segments_info"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "7d881f87",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|█████████████████████████████████████████████| 997310/997310 [09:18<00:00, 1785.12it/s]\n"
]
}
],
"source": [
"with open(\"/data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl\", \"w\") as fw:\n",
" with open(\"/data2/suno/data/harvest/youtube/pipeline/verified_metas.jsonl\") as f:\n",
" for line in tqdm.tqdm(f, total=997310):\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" segments_info = create_segments(m[\"annotated_lines\"], m[\"lang_code\"])\n",
" if len(segments_info) == 0:\n",
" continue\n",
" m[\"segments_info\"] = segments_info\n",
" m[\"uid\"] = str(uuid.uuid4())\n",
" fw.write(json.dumps(m) + \"\\n\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c73ad8b1",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "aecaa092",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "64f4df8b",
"metadata": {},
"source": [
"## analyze segments"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "cafab3be",
"metadata": {},
"outputs": [],
"source": [
"from suno_utils.audio import Audio"
]
},
{
"cell_type": "code",
"execution_count": 66,
"id": "eed5d078",
"metadata": {},
"outputs": [],
"source": [
"# !ls -lah /data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl"
]
},
{
"cell_type": "code",
"execution_count": 79,
"id": "f6580d4d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████▉| 997065/997310 [02:21<00:00, 7062.31it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"99706 samples pulled\n"
]
}
],
"source": [
"import random\n",
"\n",
"random.seed(6006)\n",
"sample_segments = []\n",
"n = 0\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl\") as f:\n",
" for line in tqdm.tqdm(f, total=997310):\n",
" n += 1\n",
" if n % 10 != 0:\n",
" continue\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" (start_s, end_s), text, uid = random.choice(m[\"segments_info\"])\n",
" sample_segments.append((uid, m[\"audio_fn\"], m[\"lang_code\"].split(\"-\")[0], (start_s, end_s), text))\n",
"# if n == 100:\n",
"# break\n",
"random.shuffle(sample_segments)\n",
"print(len(sample_segments), \"samples pulled\")"
]
},
{
"cell_type": "code",
"execution_count": 82,
"id": "ebe4da6d",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|████████████████████████████████████████████▉| 997065/997310 [02:24<00:00, 6912.17it/s]\n"
]
},
{
"data": {
"text/plain": [
"{'success': True,\n",
" 'id': 'QY17BXuWPcs',\n",
" 'lang_code': 'en-US',\n",
" 'vtt_fn': 'QY17BXuWPcs.en-US.vtt',\n",
" 'audio_fn': 'QY17BXuWPcs.webm',\n",
" 'duration_s': 241,\n",
" 'annotated_lines': [[[0.09, 5.09], 'Are you legitimately concerned'],\n",
" [[5.32, 9.68], 'for the well-being of your grandson'],\n",
" [[9.68, 14.576], 'being raised by his biological mother'],\n",
" [[14.576, 19.576], 'to the point that you fear for his safety'],\n",
" [[21.034, 24.21], \"and therefore, you think it's actionable\"],\n",
" [[24.21, 27.28], 'and he should be removed from that home?'],\n",
" [[27.28, 28.54], \"I've thought about it.\"],\n",
" [[28.54, 32.517], \"If you do think that and you don't report it,\"],\n",
" [[34.11, 38.29], \"then that's negligence on your part.\"],\n",
" [[38.29, 39.123], 'Right.'],\n",
" [[39.123, 40.83], 'As a grandparent.'],\n",
" [[40.83, 44.84], \"If you don't think it and you're just using it\"],\n",
" [[44.84, 49.83], 'as ammunition to try and fortify your position'],\n",
" [[49.83, 54.83], 'and weaken hers, that is very easy to determine'],\n",
" [[55.37, 58.71], 'and you can wind up not only having'],\n",
" [[58.71, 61.89], 'your rights limited, but terminated.'],\n",
" [[61.89, 63.63], 'Wait, we were never gonna accuse her'],\n",
" [[63.63, 67.28], 'of being any type of thing in court.'],\n",
" [[67.28, 69.79], 'She just did on videotape on national television.'],\n",
" [[69.79, 70.623], 'Yeah, thank you.'],\n",
" [[70.623, 73.24], 'We have reason to believe that you possibly do'],\n",
" [[73.24, 74.96], \"because that's what you were taught from your mother.\"],\n",
" [[74.96, 75.793], 'Uh huh.'],\n",
" [[75.793, 78.244], 'As my nephew got older, we were going to,'],\n",
" [[78.244, 81.76],\n",
" \"if it was a physical situation and he's old enough to be\"],\n",
" [[81.76, 85.65], 'literally saying, my mom hits me, my mom hurts me,'],\n",
" [[85.65, 87.18], \"then yeah, we're gonna call authorities.\"],\n",
" [[87.18, 89.73],\n",
" \"But he's little, we don't wanna just assume these things —\"],\n",
" [[89.73, 90.97], \"Well, let's take a look at how Charie tries\"],\n",
" [[90.97, 94.295], \"to investigate her grandson's bruise.\"],\n",
" [[94.295, 96.66], \"How'd you get the boo-boo on your face?\"],\n",
" [[96.66, 99.38], 'Because I got hurt.'],\n",
" [[99.38, 100.88], \"How'd you get hurt?\"],\n",
" [[102.725, 103.668], \"I don't know.\"],\n",
" [[103.668, 104.882], 'How did it happen?'],\n",
" [[104.882, 106.57], \"Hm, let's see.\"],\n",
" [[106.57, 109.33], 'Who hurt you right there, who did that?'],\n",
" [[109.33, 110.208], 'Momma.'],\n",
" [[110.208, 111.447], 'Why?'],\n",
" [[111.447, 113.908], 'Because Momma hurt me.'],\n",
" [[113.908, 117.25], \"That's not nice, I don't want Momma hurting you.\"],\n",
" [[117.25, 120.43], \"Momma's smacking on my face.\"],\n",
" [[120.43, 121.878], 'She smacked you in the face?'],\n",
" [[121.878, 123.03], 'Yeah.'],\n",
" [[123.03, 124.783], \"You don't let Momma do that.\"],\n",
" [[124.783, 128.46], 'You tell your grandma and you tell Nana, okay?'],\n",
" [[128.46, 129.54], 'Okay.'],\n",
" [[129.54, 132.61], \"I don't like that.\"],\n",
" [[132.61, 134.01], 'I protect you, okay?'],\n",
" [[134.01, 134.843], 'Okay.'],\n",
" [[134.843, 136.325], 'Give me a kiss.'],\n",
" [[136.325, 138.822], 'I love you.'],\n",
" [[138.822, 141.9], 'I guarantee you, if Alexa,'],\n",
" [[141.9, 143.93], \"and I'll tell both of you right now,\"],\n",
" [[143.93, 148.89], 'if either one of my children ever, ever, ever'],\n",
" [[148.89, 152.86], 'beat their children, I would be the first person,'],\n",
" [[152.86, 155.84], 'the very first person to turn them in.'],\n",
" [[155.84, 157.254], 'So why do you do it to your own then?'],\n",
" [[157.254, 159.008], \"She doesn't even live in my house.\"],\n",
" [[159.008, 161.42], 'Because you physically attacked her all the time.'],\n",
" [[161.42, 163.85],\n",
" \"You guys always argue, I wouldn't live in your house either.\"],\n",
" [[163.85, 165.69], 'You know when we were arguing?'],\n",
" [[165.69, 167.75], 'After this whole incident happened'],\n",
" [[167.75, 171.29], 'and Alexa was extremely confused.'],\n",
" [[171.29, 173.86],\n",
" \"My nephew was saying that he's getting hit in his face.\"],\n",
" [[173.86, 177.714],\n",
" \"He fell out of a shopping cart but you didn't believe us.\"],\n",
" [[177.714, 180.099], \"You told us that too but you didn't believe it.\"],\n",
" [[180.099, 181.119], 'Yes, I was there!'],\n",
" [[181.119, 183.427], 'I was there!'],\n",
" [[183.427, 184.26], 'You said you had your back turned'],\n",
" [[184.26, 185.093], \"and you really didn't know.\"],\n",
" [[185.093, 186.145], 'I was there when it happened.'],\n",
" [[186.145, 190.28], 'Because she is physical with her daughter.'],\n",
" [[190.28, 191.15], 'Uh huh, yes.'],\n",
" [[191.15, 192.31], 'All the time, yes.'],\n",
" [[192.31, 193.84], 'Who knows how her childhood was.'],\n",
" [[193.84, 197.423], 'What I did hear one time, it was last Christmas.'],\n",
" [[197.423, 201.09],\n",
" 'My nephew slapped her in her face, which he did a lot,'],\n",
" [[201.09, 204.107],\n",
" \"and she would laugh in front of us like oh it's so funny.\"],\n",
" [[204.107, 207.78], 'She put him in a timeout and she asked him,'],\n",
" [[207.78, 209.12], 'why did you hit mom in the face?'],\n",
" [[209.12, 210.76], \"And he said, 'cause you hit me in mine.\"],\n",
" [[210.76, 212.21], 'And she manipulated his words'],\n",
" [[212.21, 213.44], \"like I didn't hear what he said.\"],\n",
" [[213.44, 216.26],\n",
" \"And she said, no you can't just hit me 'cause you want to.\"],\n",
" [[216.26, 217.49], 'And I was disgusted.'],\n",
" [[217.49, 219.773], 'And I went into the kitchen and I told my mom.'],\n",
" [[219.773, 224.773],\n",
" 'She just manipulated what he, I heard with my own ears,'],\n",
" [[225.24, 227.003], \"what he just said and I'm sick.\"],\n",
" [[228.18, 229.653], \"I don't know what to do.\"],\n",
" [[231.0, 232.63], \"But he's a little boy.\"],\n",
" [[232.63, 234.48], \"I don't want to take him away from his mom.\"],\n",
" [[234.48, 235.853], 'And you never will.']],\n",
" 'segments_info': [[[0.09, 19.576],\n",
" 'Are you legitimately concerned for the well-being of your grandson being raised by his biological mother to the point that you fear for his safety',\n",
" 'e68bd645-45af-42b3-b225-a9307e6609c1'],\n",
" [[21.034, 40.83],\n",
" \"and therefore, you think it's actionable and he should be removed from that home? I've thought about it. If you do think that and you don't report it, then that's negligence on your part. Right. As a grandparent.\",\n",
" 'fef40cf2-bd75-4730-a292-6c9bbb40a61d'],\n",
" [[40.83, 58.71],\n",
" \"If you don't think it and you're just using it as ammunition to try and fortify your position and weaken hers, that is very easy to determine and you can wind up not only having\",\n",
" 'c5220f08-4e8d-4a83-b120-1725395154ec'],\n",
" [[58.71, 78.244],\n",
" \"your rights limited, but terminated. Wait, we were never gonna accuse her of being any type of thing in court. She just did on videotape on national television. Yeah, thank you. We have reason to believe that you possibly do because that's what you were taught from your mother. Uh huh. As my nephew got older, we were going to,\",\n",
" '1bbf0fb6-e356-4dfb-a2e4-1f466ffd9e4c'],\n",
" [[78.244, 96.66],\n",
" \"if it was a physical situation and he's old enough to be literally saying, my mom hits me, my mom hurts me, then yeah, we're gonna call authorities. But he's little, we don't wanna just assume these things — Well, let's take a look at how Charie tries to investigate her grandson's bruise. How'd you get the boo-boo on your face?\",\n",
" '5e58b62a-1b11-4076-b3b5-f074cc865703'],\n",
" [[96.66, 113.908],\n",
" \"Because I got hurt. How'd you get hurt? I don't know. How did it happen? Hm, let's see. Who hurt you right there, who did that? Momma. Why? Because Momma hurt me.\",\n",
" 'fb5d2e1f-6090-4ad0-b9de-8b7380527de8'],\n",
" [[113.908, 132.61],\n",
" \"That's not nice, I don't want Momma hurting you. Momma's smacking on my face. She smacked you in the face? Yeah. You don't let Momma do that. You tell your grandma and you tell Nana, okay? Okay. I don't like that.\",\n",
" 'b572cd69-7202-4cb3-8975-f82b3b6acde5'],\n",
" [[132.61, 148.89],\n",
" \"I protect you, okay? Okay. Give me a kiss. I love you. I guarantee you, if Alexa, and I'll tell both of you right now, if either one of my children ever, ever, ever\",\n",
" 'f5c63973-811d-424b-820c-ece88dda3729'],\n",
" [[148.89, 167.75],\n",
" \"beat their children, I would be the first person, the very first person to turn them in. So why do you do it to your own then? She doesn't even live in my house. Because you physically attacked her all the time. You guys always argue, I wouldn't live in your house either. You know when we were arguing? After this whole incident happened\",\n",
" 'c6585c38-f30e-45f8-b164-5438c185ec36'],\n",
" [[167.75, 186.145],\n",
" \"and Alexa was extremely confused. My nephew was saying that he's getting hit in his face. He fell out of a shopping cart but you didn't believe us. You told us that too but you didn't believe it. Yes, I was there! I was there! You said you had your back turned and you really didn't know. I was there when it happened.\",\n",
" 'cef53809-51c0-4d8e-a5d1-b6d49b312e19'],\n",
" [[186.145, 204.107],\n",
" \"Because she is physical with her daughter. Uh huh, yes. All the time, yes. Who knows how her childhood was. What I did hear one time, it was last Christmas. My nephew slapped her in her face, which he did a lot, and she would laugh in front of us like oh it's so funny.\",\n",
" 'bbf927ce-0a73-4d6f-ad02-3494b4e6c07d'],\n",
" [[204.107, 219.773],\n",
" \"She put him in a timeout and she asked him, why did you hit mom in the face? And he said, 'cause you hit me in mine. And she manipulated his words like I didn't hear what he said. And she said, no you can't just hit me 'cause you want to. And I was disgusted. And I went into the kitchen and I told my mom.\",\n",
" '75b0509d-ec45-452a-9ca5-270be6791634'],\n",
" [[219.773, 235.853],\n",
" \"She just manipulated what he, I heard with my own ears, what he just said and I'm sick. I don't know what to do. But he's a little boy. I don't want to take him away from his mom. And you never will.\",\n",
" '7f6fa1cf-2e6e-4904-be5a-cbb0f86f30e0']]}"
]
},
"execution_count": 82,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"with open(\"/data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl\") as f:\n",
" for line in tqdm.tqdm(f, total=997310):\n",
" n += 1\n",
" if n % 10 != 0:\n",
" continue\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" if m[\"id\"] == \"IqJnfh42LVA\":\n",
" break\n",
"m"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b3fba280",
"metadata": {},
"outputs": [],
"source": [
"# difference of more than 2 seconds between below is issue\n",
"# Audio.from_file(\"IqJnfh42LVA.m4a\").duration_s\n",
"# Audio.get_details(\"IqJnfh42LVA.m4a\")[\"duration_s\"]\n",
"\n",
"# --\n",
"\n",
"# wrong audio language\n",
"# RgRGM5KGO4I.m4a - tr - 80.608 97.047\n",
"# vw4V3XPVCFw.webm - en - 208.08 225.86\n",
"# dE4WiTX9onA.webm - en - 2.802 21.567\n",
"# APOWkWO8Lmw.m4a - ru - 137.149 154.199\n",
"\n",
"# silence\n",
"# VvbRRf9fRjg.m4a - en - 201.9 216.259 -- instruction video\n",
"# QFu8skFZuGU.webm - de - 19.64 38.1 -- instruction video\n",
"# eeIfiqZvd0M.m4a - en - 65.597 67.597 -- comments\n",
"# j8PyJQRdBSY.m4a - en - 86.36 104.6 -- instruction video\n",
"\n",
"# boundary mistakes\n",
"# UINV1-52A_E.m4a - en - 9.69 29.28\n",
"# IqJnfh42LVA.m4a - en - 38.471 57.49 -- conversion issue\n",
"\n",
"# wrong\n",
"# AkKJcdFrI9U.webm - en - 160.08 163.32"
]
},
{
"cell_type": "code",
"execution_count": 191,
"id": "eccd0515",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"AkKJcdFrI9U.webm - en - 160.08 163.32\n",
"Good effort. BIOVISJON AS Trygve Lie\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uitx0kXjIO0.m4a - zh - 444.3 463.866\n",
"贫苦的人们被逼得铤而走险打家劫舍而在云省这个以土地贫濯而出了名的省份内人们却还能吃饱安居德伦向紫川秀解释说魔族的徵粮队不怎么来云省一来这里的民风剽悍为了五公斤麦子这里的半兽人可以拚命；二来这里的土地贫瘠\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"f-vodjl7FVE.m4a - en - 146.1 162.739\n",
"The pieces won't keep rotating around and giving you a hard time. If you'll be using wooden skewers then please remember to soak them first for about an hour before using. I alternated the chicken pieces with onions. You could add whatever vegetables you prefer to your kebabs but personally I prefer them\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GD3JS1uLAaE.webm - zh - 329.71 348.66\n",
"大海航行靠舵手万物生长靠太阳还用那种思维方式的话第一你在加拿大很难生存下去第二甚至你在移民之前你都不可能成功所以第一步第一步最重要的就是改变你的思维方式把你以前的东西全都给忘掉\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"eLDsT4CuR10.m4a - en - 93.31 109.92\n",
"use their HRH titles. Retail expert Andy Barr believes Harry and Meghan's new Sussex Royal product empire could help them rake in revenues of £400million.But this means other brands will not be able to\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"7Yryksm0i8o.m4a - en - 163.689 182.889\n",
"tracks, and businesses in industry boomed. But this was not all! In 1873, Andrew Halliday — a British immigrant to the United States — invented cable cars. They\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"fAPZsaJxwg8.m4a - da - 3.9 22.9\n",
"Du skal vælge din behandling hos EuroEyes, fordi vores øjenlæger og kirurger — | har rigtig mange års erfaring. Vi har kun high volumekirurger — | og det vil sige kirurger, som gennemfører rigtig mange operationer. Vi har et bredt udvalg af løsninger. Derfor tilbyder vi målrettede behandlinger —\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"TohG7F8M3Ls.webm - en - 751.569 769.4\n",
"Right over the side, just like a waterfall sorta, only this is snow. Just let it fall right down through there. There, just sort of blend all that together. I'll grab the little blender brush and smooth it out real smooth. There, see?\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"GUpelheHyAU.webm - ko - 74.074 93.96\n",
"입자선, 또는 전자기파로 모든 세포를 죽일 수 있는 매우 위험한 물질이고 색도 없고, 온도도 없고, 아무런 느낌도 없습니다. 방사선치료 중 상상해보십시오. 방사선이 매우 따뜻하고 밝은 빛이며 이 빛이 내 몸 안의 암 덩어리에 닿으면 마치 햇빛이 아이스크림을 녹이듯이\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"05EpaMonJHA.webm - nl - 29.2 44.569\n",
"Er zijn 3 modellen leverbaar die alle standaard uitgerust zijn met een elektronisch codeslot. Met 1 uur brandveiligheid biedt de kluis een uitstekende oplossing voor het opbergen van papier, zoals documenten, certificaten en contracten.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"MV0AGrX3FY4.webm - en - 873.68 882.88\n",
"his rapping and you know being real streety in the way he presented stuff that actually brings this stuff to life with his character and just very typical of the character of him. So let's continue.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"TZ5roLy087s.m4a - en - 2440.35 2457.9\n",
"me their passwords, or unlock their phones, and, you know, things like that. And it's like, well, wait a minute. Yeah, that's your device. I mean, if you don't establish those rules from the very beginning, it's real tough to then, you know, recur out that, yeah. And then the grandparents who times you know, economically, they're asked to step up for different\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"ncsustS7_ew.webm - pt - 111.049 126.999\n",
"Aos já inscritos | Gratidão pelas inscrições, comentários, curtidas e sejam bem-vindos de volta. Verifique se continua inscrito e acione a campainha, também dê seu like neste e em outros vídeos deste canal, e faça comentário, nem que for só Oi.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"APOWkWO8Lmw.m4a - ru - 137.149 154.199\n",
"После взлёта мне придётся хранить радиомолчание, и я не смогу передавать коды в штаб. Поспеши. Гейб, меня обнаружили, но я не улечу, пока ты не передашь мне все коды с ракет. Я буду сдерживать их столько, сколько смогу. Гейб, кажется, они глушат мне связь, но я получила последний код.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"UM97_u1WBfQ.webm - en - 183.12 196.48\n",
"a blanket over his head and wouldn't engage with the officers interviewing him.\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"n8hF7WXJmjI.m4a - ko - 129.637 139.724\n",
"'돌려내'라고 한탄해 봤자 '더는 싫어'라고 울어 봤자 알까 보냐, 그것이 운명이다 심하게 무르고 하찮은 나날이 진짜잖아\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Mz4A5FudBSo.m4a - th - 188.11 206.76\n",
"ชุดนี้เนี่ยถือว่าเป็นมาสเตอร์พีซของวงการตลกเลยนะครับบางคนฟังจนเทปยืดเทปยานกันเลยทีเดียวแต่มาฟังตอนนี้เด็กรุ่นใหม่ๆก็อาจจะไม่ค่อยตลกแล้วก็ได้เพราะว่ายุคสมัยทางภาษาและวัฒนธรรมก็เปลี่ยนไปเด๋อดู๋ดี๋ดอนแดงเป็นชื่อของสมาชิกคณะในเทปชุดนี้ครับ\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"_BbzLXb_vm4.webm - en - 131.98 150.9\n",
"that Kim Kardashian has to be aware of because she added an additional K in her product name to differentiate hers from Lori Harvey's skincare line. Kim Kardashian announced the name of her skincare line this morning and people are already reacting to her latest antic with her skincare line being named\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"ZxaBX5LizSo.webm - en - 35.82 49.14\n",
"I get one done in a second if you wait Sometimes I think sitting on trains Every stop I get to, I'm clocking that game Everyone's a winner, we're making our fame Bona fide hustler making my name\n",
"----------\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"4pIHt23D04E.m4a - en - 0.763 18.551\n",
"Travel is on mind of many, as airport travel restrictions loosen up today following the pandemc. With many more planning to travel the need for aircraft staff is dire. Here to tell us more about how you can be part of the solution is Skip DuRand AMT Director from California Aeronautical University. Thanks for bein here Skip. Hi, thank you for having me.\n",
"----------\n"
]
}
],
"source": [
"for uid, audio_fn, lang_code, (start_s, end_s), text in sample_segments[45:65]:\n",
" Audio.from_file(os.path.join(DATA_DIR, \"audio\", audio_fn)).play(start_s, end_s)\n",
" print(audio_fn, \"-\", lang_code, \"-\", start_s, end_s)\n",
" print(text)\n",
" print(\"-\"*10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d26b3aa0",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "ee3e6bc4",
"metadata": {},
"source": [
"ZxaBX5LizSo.webm - en - 35.82 49.14\n",
"I get one done in a second if you wait Sometimes I think sitting on trains Every stop I get to, I'm clocking that game Everyone's a winner, we're making our fame Bona fide hustler making my name"
]
},
{
"cell_type": "code",
"execution_count": 195,
"id": "b5054fb9",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ncsustS7_ew.pt-BR.vtt\r\n",
"ncsustS7_ew.webm\r\n"
]
}
],
"source": [
"!ls /data2/suno/data/harvest/youtube/audio/ | grep ncsustS7"
]
},
{
"cell_type": "code",
"execution_count": 197,
"id": "c7ba56a4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WEBVTT\r\n",
"Kind: captions\r\n",
"Language: en\r\n",
"\r\n",
"00:00:16.010 --> 00:00:25.660\r\n",
"I fly like paper, get high like planes\r\n",
"If you catch me at the border, I got visas\r\n",
"\r\n",
"00:00:25.660 --> 00:00:27.870\r\n",
"in my name\r\n",
"If you come around here, I make 'em all day\r\n",
"\r\n",
"00:00:27.870 --> 00:00:28.870\r\n",
"I get one done in a second if you wait\r\n",
"\r\n",
"00:00:28.870 --> 00:00:34.820\r\n",
"I fly like paper, get high like planes\r\n",
"If you catch me at the border, I got visas\r\n",
"\r\n",
"00:00:34.820 --> 00:00:35.820\r\n",
"in my name\r\n",
"If you come around here, I make 'em all day\r\n",
"\r\n",
"00:00:35.820 --> 00:00:38.670\r\n",
"I get one done in a second if you wait\r\n",
"\r\n",
"00:00:38.670 --> 00:00:45.410\r\n",
"Sometimes I think sitting on trains\r\n",
"Every stop I get to, I'm clocking that game\r\n",
"\r\n",
"00:00:45.410 --> 00:00:49.140\r\n",
"Everyone's a winner, we're making our fame\r\n",
"Bona fide hustler making my name\r\n",
"\r\n",
"00:00:49.140 --> 00:01:05.409\r\n",
"Sometimes I think sitting on trains\r\n",
"Every stop I get to, I'm clocking that game\r\n",
"\r\n",
"00:01:05.409 --> 00:01:22.660\r\n",
"Everyone's a winner now we're making our fame\r\n",
"Bona fide hustler making my name\r\n",
"\r\n",
"00:01:22.660 --> 00:01:34.290\r\n",
"All I wanna do is (BANG BANG BANG BANG )\r\n",
"And a,((KACHENG)) and take your money\r\n",
"\r\n",
"00:01:34.290 --> 00:01:42.860\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a, (KACHENG)and take your money\r\n",
"\r\n",
"00:01:42.860 --> 00:01:53.770\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a, (KACHENG)and take your money\r\n",
"\r\n",
"00:01:53.770 --> 00:01:57.610\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:01:57.610 --> 00:01:59.670\r\n",
"Pirate skulls and bones\r\n",
"Sticks and stones and weed and bones\r\n",
"\r\n",
"00:01:59.670 --> 00:02:01.530\r\n",
"Running when we hit 'em\r\n",
"Lethal poison through their system\r\n",
"\r\n",
"00:02:01.530 --> 00:02:03.590\r\n",
"Pirate skulls and bones\r\n",
"Sticks and stones and weed and bones\r\n",
"\r\n",
"00:02:03.590 --> 00:02:05.450\r\n",
"Running when we hit 'em\r\n",
"Lethal poison through their system\r\n",
"\r\n",
"00:02:05.450 --> 00:02:08.440\r\n",
"No one on the corner has swagger like us\r\n",
"Hit me on my burner, prepaid wireless\r\n",
"\r\n",
"00:02:08.440 --> 00:02:11.240\r\n",
"We pack and deliver like UPS trucks\r\n",
"Already going to hell, just pumping that gas\r\n",
"\r\n",
"00:02:11.240 --> 00:02:14.230\r\n",
"No one on the corner has swagger like us\r\n",
"Hit me on my burner, prepaid wireless\r\n",
"\r\n",
"00:02:14.230 --> 00:02:17.030\r\n",
"We pack and deliver like UPS trucks\r\n",
"Already going to hell, just pumping that gas\r\n",
"\r\n",
"00:02:17.030 --> 00:02:19.640\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:19.640 --> 00:02:22.250\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:22.250 --> 00:02:24.870\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:24.870 --> 00:02:27.480\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:27.480 --> 00:02:32.510\r\n",
"M.I.A., third world democracy\r\n",
"Yeah, I got more records than the KGB\r\n",
"\r\n",
"00:02:32.510 --> 00:02:33.510\r\n",
"So, uh, no funny business, are you all ready?\r\n",
"\r\n",
"00:02:33.510 --> 00:02:34.510\r\n",
"Some, some, some I, some I murder\r\n",
"Some I, some I let go\r\n",
"\r\n",
"00:02:34.510 --> 00:02:35.510\r\n",
"Some, some, some I, some I murder\r\n",
"Some I, some I let go\r\n",
"\r\n",
"00:02:35.510 --> 00:02:36.510\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:36.510 --> 00:02:37.510\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:37.510 --> 00:02:38.510\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:38.510 --> 00:02:39.510\r\n",
"All I wanna do is(BANG BANG BANG BANG )\r\n",
"And a,(KACHENG) and take your money\r\n",
"\r\n",
"00:02:39.510 --> 00:02:40.510\r\n",
"Read more: M.I.A.\r\n",
"\r\n",
"00:02:40.510 --> 00:02:40.514\r\n",
"- Paper Planes Lyrics | MetroLyrics\r\n",
"\r\n"
]
}
],
"source": [
"!cat /data2/suno/data/harvest/youtube/audio/ZxaBX5LizSo.en.vtt"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d1d44f10",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "51523a35",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "1e6c0fdf",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "25a4d774",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "37138b86",
"metadata": {},
"source": [
"### extra filter for asr english?"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cee91c04",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "1bf0cb96",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "e239c798",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "2549db4f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "769dcecc",
"metadata": {},
"source": [
"### move segments to s3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "964daf0a",
"metadata": {},
"outputs": [],
"source": [
"# restrict to certain languages\n",
"# subsample based on language prominence to get ~20k hours"
]
},
{
"cell_type": "code",
"execution_count": 132,
"id": "23d24c9c",
"metadata": {},
"outputs": [],
"source": [
"import tempfile\n",
"import subprocess\n",
"import funcy\n",
"\n",
"import boto3\n",
"import numpy as np\n",
"\n",
"from suno_utils.audio import Audio\n",
"from suno_utils.utils.numbers import safe_round\n",
"\n",
"S3_BUCKET = \"suno-data\"\n",
"S3_PATH = \"datasets/harvest/youtube/dataset\"\n",
"\n",
"MAX_SEGMENTS_PER_FILE = 100\n",
"\n",
"def write_segments(line, suppress_errors=True):\n",
" try:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" return None\n",
" m = json.loads(line)\n",
" # ~150x realtime to load audio\n",
" audio_fp = os.path.join(DATA_DIR, \"audio\", m[\"audio_fn\"])\n",
" true_duration_s = Audio.from_file(audio_fp, sample_rate=16_000, byte_width=2).duration_s\n",
" label_duration_s = Audio.get_duration_s(audio_fp, attempt_using_header=True)\n",
" if np.abs(true_duration_s - label_duration_s) > 2:\n",
" return None\n",
" with tempfile.TemporaryDirectory() as tmp_dir:\n",
" # ~80x realtime to encode entire audio, ~14x segment-by-segment\n",
" # ~30x if done in single command\n",
" tmp_fp = os.path.join(tmp_dir, \"tmp.opus\")\n",
" out = subprocess.call(\n",
" f\"ffmpeg -y -i {audio_fp} -c:a libopus -b:a 32K -ac 1 {tmp_fp}\".split(), \n",
" stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n",
" )\n",
" convert_cmd_lines = [\n",
" f\"ffmpeg -y -i {tmp_fp}\"\n",
" ]\n",
" segments_info = m[\"segments_info\"][:]\n",
" random.shuffle(segments_info)\n",
" segments_info = sorted(segments_info[:MAX_SEGMENTS_PER_FILE], key=lambda x: x[0][0])\n",
" segment_metas = []\n",
" for (start_s, end_s), text, segment_uid in segments_info:\n",
" to_fn = f\"{segment_uid}.opus\"\n",
" to_fp = os.path.join(tmp_dir, to_fn)\n",
"# audio_segment = audio.get_slice(start_s, end_s)\n",
"# audio_segment.to_opus(to_fp, opusenc_path=\"/bin/opusenc\")\n",
" duration_s = end_s - start_s\n",
" convert_cmd_lines.append(\n",
" f\"-c copy -t {round(duration_s, 2)} -ss {round(start_s, 2)} {to_fp}\"\n",
" )\n",
" segment_metas.append({\n",
" \"original_id\": m[\"id\"],\n",
" \"original_uid\": m[\"uid\"],\n",
" \"original_offset_s\": safe_round(start_s),\n",
" \"id\": segment_uid,\n",
" \"lang_code\": m[\"lang_code\"],\n",
" \"audio_filename\": to_fn,\n",
" \"audio_filepath\": to_fp,\n",
" \"duration_s\": safe_round(duration_s),\n",
" \"text\": text,\n",
" })\n",
" if len(segment_metas) == 0:\n",
" return None\n",
" out = subprocess.call(\n",
" \" \".join(convert_cmd_lines).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n",
" )\n",
" assert(out == 0) \n",
" # upload to s3\n",
" s3 = boto3.resource(\"s3\")\n",
" base_lang = m[\"lang_code\"].split(\"-\")[0]\n",
" for m in segment_metas:\n",
" to_fp = os.path.join(S3_PATH, f\"{base_lang}/{m['original_uid']}/{m['audio_filename']}\")\n",
" s3.Bucket(S3_BUCKET).upload_file(m[\"audio_filepath\"], to_fp)\n",
" os.remove(m[\"audio_filepath\"])\n",
" m[\"audio_filepath\"] = to_fp\n",
" return segment_metas\n",
" except Exception as e:\n",
" if not suppress_errors:\n",
" raise e\n",
" return None"
]
},
{
"cell_type": "code",
"execution_count": 129,
"id": "61489405",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 769 ms, sys: 1.18 s, total: 1.95 s\n",
"Wall time: 19.2 s\n"
]
}
],
"source": [
"# %%time\n",
"# out = write_segments(line, suppress_errors=False)"
]
},
{
"cell_type": "code",
"execution_count": 131,
"id": "853b8231",
"metadata": {},
"outputs": [],
"source": [
"# aws s3 rm --recursive s3://suno-data/datasets/harvest/youtube/dataset/"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c5c6a6ed",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 34%|███████████████▏ | 67/199 [57:00:51<111:26:29, 3039.32s/it]"
]
}
],
"source": [
"# ~7 days for entire 100k hours\n",
"# 5k files - 4.9k working, 137k segments, ~630h of data -- ~50mins on 32 cores\n",
"n_cores = 31\n",
"chunksize = 5000\n",
"random.seed(6006)\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/upload_segments_meta.jsonl\", \"w\") as fw:\n",
" fw.write(\"\")\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl\") as f:\n",
" for n, lines_chunk in tqdm.tqdm(\n",
" enumerate(funcy.chunks(chunksize, f.readlines())), total=997_065//chunksize,\n",
" ):\n",
" with Pool(n_cores) as p:\n",
" out = p.map(write_segments, lines_chunk, chunksize=1)\n",
" out_write = [m for m in out if m is not None]\n",
" with open(\"/data2/suno/data/harvest/youtube/pipeline/upload_segments_meta.jsonl\", \"a\") as fw:\n",
" for m in out_write:\n",
" for mm in m:\n",
" fw.write(json.dumps(mm) + \"\\n\")\n",
"# break"
]
},
{
"cell_type": "code",
"execution_count": 147,
"id": "49ddac6c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"112353.75 hours of data\n"
]
}
],
"source": [
"# get durations\n",
"durations_s = []\n",
"with open(\"/data2/suno/data/harvest/youtube/pipeline/upload_segments_meta.jsonl\") as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if len(line) == 0:\n",
" continue\n",
" m = json.loads(line)\n",
" durations_s.append(m[\"duration_s\"])\n",
"print(round(np.sum(durations_s) / 60 / 60, 2), \"hours of data\")\n",
"# 112_350 hours"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c0041423",
"metadata": {},
"outputs": [],
"source": [
"# !s3cmd put /data2/suno/data/harvest/youtube/pipeline/upload_segments_meta.jsonl s3://suno-data/datasets/harvest/youtube/"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "751a235f",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "42b86a43",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "d4a62d4c",
"metadata": {},
"source": [
"## Playground"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ddd22c1d",
"metadata": {},
"outputs": [],
"source": [
"vtt_fns = os.listdir(os.path.join(DATA_DIR, \"audio\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9ee50a7a",
"metadata": {},
"outputs": [],
"source": [
"l = [fn.split(\".\")[0] for fn in vtt_fns]\n",
"vc = pd.Series([e for e in l]).value_counts()\n",
"unique_ids = set(vc[vc == 1].index)"
]
},
{
"cell_type": "code",
"execution_count": 125,
"id": "97d9c1c4",
"metadata": {},
"outputs": [],
"source": [
"langs = [fn.split(\".\")[1] for fn in vtt_fns if fn.split(\".\")[0] in unique_ids]\n",
"vc = pd.Series(langs).value_counts()"
]
},
{
"cell_type": "code",
"execution_count": 126,
"id": "6194bfa5",
"metadata": {},
"outputs": [],
"source": [
"# can we tell \"KIgOUc63dRo\" is russian?"
]
},
{
"cell_type": "code",
"execution_count": 183,
"id": "155a8129",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[youtube] rKYaV-vRc7E: Downloading webpage\n",
"[download] Destination: tmp/rKYaV-vRc7E.webm\n",
"[download] 100% of 1.28MiB in 00:0091MiB/s ETA 00:001\n"
]
}
],
"source": [
"import youtube_dl\n",
"\n",
"YDL_OPTS = {\n",
" \"format\": \"worstaudio\",\n",
" \"outtmpl\": \"tmp/%(id)s.%(ext)s\",\n",
" \"writesubtitles\": True,\n",
" \"allsubtitles\": True,\n",
" \"subtitlesformat\": \"best\",\n",
" \"socket_timeout\": 5.0,\n",
"}\n",
"\n",
"YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n",
"\n",
"# url = YOUTUBE_BASE_URL + \"KIgOUc63dRo\" # russian, all sorts of stuff\n",
"\n",
"# url = YOUTUBE_BASE_URL + \"n9pMNI-tWzE\" # cnn, english only\n",
"# len(info[\"subtitles\"].keys()) == 1\n",
"\n",
"url = YOUTUBE_BASE_URL + \"rKYaV-vRc7E\" # last week tonight, auto english only\n",
"\n",
"with youtube_dl.YoutubeDL(YDL_OPTS) as ydl:\n",
" info = ydl.extract_info(url, download=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2f59dd5f",
"metadata": {},
"outputs": [],
"source": [
"# TODO: next look at how many have ALL vs just a few in 'manual' category"
]
},
{
"cell_type": "code",
"execution_count": 176,
"id": "575ac7ad",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 176,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(info[\"subtitles\"].keys()) == 1"
]
},
{
"cell_type": "code",
"execution_count": 152,
"id": "fcc92b48",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"dict_keys(['af', 'ak', 'sq', 'am', 'ar', 'hy', 'as', 'ay', 'az', 'bn', 'eu', 'be', 'bho', 'bs', 'bg', 'my', 'ca', 'ceb', 'zh-Hans', 'zh-Hant', 'co', 'hr', 'cs', 'da', 'dv', 'nl', 'en', 'eo', 'et', 'ee', 'fil', 'fi', 'fr', 'gl', 'lg', 'ka', 'de', 'el', 'gn', 'gu', 'ht', 'ha', 'haw', 'iw', 'hi', 'hmn', 'hu', 'is', 'ig', 'id', 'ga', 'it', 'ja', 'jv', 'kn', 'kk', 'km', 'rw', 'ko', 'kri', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mi', 'mr', 'mn', 'ne', 'nso', 'no', 'ny', 'or', 'om', 'ps', 'fa', 'pl', 'pt', 'pa', 'qu', 'ro', 'ru', 'sm', 'sa', 'gd', 'sr', 'sn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'su', 'sw', 'sv', 'tg', 'ta', 'tt', 'te', 'th', 'ti', 'ts', 'tr', 'tk', 'uk', 'und', 'ur', 'ug', 'uz', 'vi', 'cy', 'fy', 'xh', 'yi', 'yo', 'zu'])"
]
},
"execution_count": 152,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"info[\"automatic_captions\"].keys()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7b16a8eb",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "2d71a354",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "55517019",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "3a85b73e",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "8ace8d00",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "9141e599",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}