{ "cells": [ { "cell_type": "code", "execution_count": 33, "id": "47a1e830", "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import tqdm\n", "import shutil\n", "\n", "import pandas as pd\n", "from multiprocessing import Pool\n", "\n", "# from suno_utils.harvest.youtube.constants_languages import USED_LANG_CODES\n", "\n", "\n", "DATA_DIR = \"/data2/suno/data/harvest/youtube\"\n", "PROJECT_DIR = os.path.join(DATA_DIR, \"en_de_translation\")\n", "\n", "shutil.rmtree(PROJECT_DIR, ignore_errors=True)\n", "os.makedirs(PROJECT_DIR)" ] }, { "cell_type": "markdown", "id": "ac6ddc2b", "metadata": {}, "source": [ "## get german/english subtitles" ] }, { "cell_type": "code", "execution_count": 2, "id": "d67ecdfb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "175904 files found\n" ] } ], "source": [ "# find ones that have <= 20 with english and german\n", "vtt_fns = [fn for fn in os.listdir(os.path.join(DATA_DIR, \"audio\")) if fn.endswith(\"vtt\")]\n", "vtt_fn_data = [\n", " (fn, fn.split(\".\")[0], fn.split(\".\")[1].split(\"-\")[0]) \n", " for fn in vtt_fns if len(fn.split(\".\")) == 3\n", "]\n", "df = pd.DataFrame(vtt_fn_data, columns=[\"fn\", \"id\", \"lang\"])\n", "df[\"n_lang\"] = df[\"id\"].map(df.groupby(\"id\")[\"lang\"].count())\n", "df = df[(df[\"n_lang\"] <= 20) & (df[\"n_lang\"]>=2)]\n", "relevant_ids = set(df[df[\"lang\"] == \"de\"][\"id\"]) & set(df[df[\"lang\"] == \"en\"][\"id\"])\n", "print(len(relevant_ids), \"files found\")\n", "df = df[df[\"id\"].isin(relevant_ids)]\n", "df = df[df[\"lang\"].isin([\"en\", \"de\"])]\n", "# note that we sort first to prefer 'en' over 'en-GB'\n", "df = df.sort_values(by=\"fn\", ascending=False).groupby([\"id\", \"lang\"]).first().reset_index()\n", "assert(df.shape[0] == len(relevant_ids) * 2)\n", "df = df.set_index([\"id\", \"lang\"]).unstack(\"lang\")[\"fn\"].reset_index()\n", "df.columns.name = None\n", "df = df.rename(columns={\"de\": \"de_fn\", \"en\": \"en_fn\"})\n", "assert(df.isnull().sum().sum() == 0)\n", "vtt_data = list(df.itertuples(index=False, name=None))" ] }, { "cell_type": "code", "execution_count": 3, "id": "187abdd0", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[('---z5aBhED0', '---z5aBhED0.de.vtt', '---z5aBhED0.en.vtt'),\n", " ('--4B0Pg4pf8', '--4B0Pg4pf8.de.vtt', '--4B0Pg4pf8.en.vtt'),\n", " ('--4VlwpePzM', '--4VlwpePzM.de.vtt', '--4VlwpePzM.en.vtt')]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vtt_data[:3]" ] }, { "cell_type": "code", "execution_count": 4, "id": "dea785a9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n" ] } ], "source": [ "import fasttext\n", "\n", "from suno_utils.harvest.youtube.captions import parse_vtt, verify_vtt_lang\n", "\n", "# !wget https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin\n", "fasttext_lang_model = fasttext.load_model(\"lid.176.bin\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "631a0cf2", "metadata": {}, "outputs": [], "source": [ "def bar(vtt_fn):\n", " with open(os.path.join(DATA_DIR, \"audio\", vtt_fn)) as f:\n", " vtt_str = f.read()\n", " try:\n", " check_lang_code = vtt_fn.split(\".\")[1]\n", " _, annotated_lines = parse_vtt(vtt_str)\n", " verify_vtt_lang(annotated_lines, check_lang_code, fasttext_lang_model)\n", " verified_vtt_fns.append(vtt_fn)\n", " return True\n", " except:\n", " pass\n", " return False\n", "\n", "def foo(vtt_info):\n", " _id, de_vtt, en_vtt = vtt_info\n", " if bar(de_vtt) and bar(en_vtt):\n", " return vtt_info\n", " return None" ] }, { "cell_type": "code", "execution_count": 53, "id": "ed2a804b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.0 hours\n" ] } ], "source": [ "# 2 hours for 175904 files (2x check)\n", "import time\n", "t0 = time.time()\n", "out = [foo(e) for e in vtt_data]\n", "t1 = time.time()\n", "print(round((t1 - t0) / 60 / 60, 1), \"hours\")\n", "# with Pool(30) as p:\n", "# out = p.map(foo, vtt_data, chunksize=1)" ] }, { "cell_type": "code", "execution_count": 54, "id": "d3fa84c8", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "162443" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "verified_vtt_fns = [e for e in out if e is not None]\n", "len(verified_vtt_fns)" ] }, { "cell_type": "code", "execution_count": 55, "id": "1326c974", "metadata": {}, "outputs": [], "source": [ "with open(\"verified_en_de_vtt_fns.json\", \"w\") as f:\n", " json.dump(verified_vtt_fns, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "e5ec4c39", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "068c2b6a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b9aae7ef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "d9bd24fa", "metadata": {}, "source": [ "## Do parsing/cleaning/text-filtering base on cap etc" ] }, { "cell_type": "code", "execution_count": 57, "id": "b1d63523", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "162443 promising ids\n" ] } ], "source": [ "import tqdm\n", "import funcy\n", "import re\n", "import string\n", "import numpy as np\n", "\n", "from suno_utils.harvest.youtube.captions import parse_vtt, clean_cc_lines, validate_quality\n", "from suno_utils.harvest.youtube.constants.base import CASED_LANGS, SUPPORTED_LANGS, NON_WHITESPACE_LANGS\n", "\n", "with open(\"verified_en_de_vtt_fns.json\") as f:\n", " vtt_fns_info = json.load(f)\n", "from collections import defaultdict\n", "RETAINED_VTTS = defaultdict(dict)\n", "for _id, de_vtt, en_vtt in vtt_fns_info:\n", " RETAINED_VTTS[_id][\"de_vtt\"] = de_vtt\n", " RETAINED_VTTS[_id][\"en_vtt\"] = en_vtt\n", "RETAINED_IDS = set(RETAINED_VTTS.keys())\n", "print(len(RETAINED_IDS), \"promising ids\")" ] }, { "cell_type": "code", "execution_count": 65, "id": "a0132983", "metadata": {}, "outputs": [], "source": [ "def _get_cc_name(m):\n", " cc_name = None\n", " for k, v in m[\"subtitles\"].items():\n", " if k.startswith(\"en\"):\n", " for e in v:\n", " if e[\"ext\"] == \"vtt\":\n", " mg = re.search(r\"name\\=([^\\&]+?)\\&\", e[\"url\"])\n", " if mg is not None:\n", " cc_name = mg.group(1)\n", " return cc_name\n", "\n", "def process_item(line):\n", " line = line.strip()\n", " if len(line) == 0:\n", " return {\"success\": False, \"error_msg\": \"empty\"}\n", " m = json.loads(line)\n", " if not m[\"success\"]:\n", " return {\"success\": False, \"error_msg\": \"harvest issue\"}\n", " if m[\"id\"] not in RETAINED_IDS:\n", " return {\"success\": False, \"error_msg\": \"file excluded\"}\n", " youtube_id = m[\"id\"]\n", "# lang_codes = [k for k in m[\"meta\"][\"subtitles\"].keys() if ]\n", "# if lang_code not in USED_LANG_CODES:\n", "# return {\"success\": False, \"error_msg\": \"language not supported\"}\n", "# vtt_fn = f\"{youtube_id}.{lang_code}.vtt\"\n", " audio_fn = m[\"audio_filename\"]\n", " duration_s = m[\"meta\"][\"duration\"]\n", " res = {\n", " \"success\": False, \n", " \"id\": youtube_id, \n", " \"vtt_fn_de\": RETAINED_VTTS[youtube_id][\"de_vtt\"],\n", " \"vtt_fn_en\": RETAINED_VTTS[youtube_id][\"en_vtt\"],\n", " \"audio_fn\": audio_fn,\n", " \"duration_s\": duration_s,\n", " }\n", "# if vtt_fn not in promising_vtt_fns:\n", "# return {\"success\": False, \"error_msg\": \"vtt filename messup\"}\n", " view_count = m[\"meta\"][\"view_count\"]\n", " if view_count < 50:\n", " res[\"error_msg\"] = \"too few views\"\n", " return res\n", " if duration_s < 20 or duration_s > 2 * 60 * 60:\n", " res[\"error_msg\"] = \"too long or short\"\n", " return res\n", " try:\n", " for vtt_fn, lang in [(res[\"vtt_fn_de\"], \"de\"), (res[\"vtt_fn_en\"], \"en\")]:\n", " with open(os.path.join(DATA_DIR, \"audio\", vtt_fn)) as f:\n", " vtt_str = f.read()\n", " if \"\" in vtt_str or \"\" in vtt_str:\n", " raise ValueError(\"word based timestamps found. likely auto-generated\")\n", " lang_code, annotated_lines = parse_vtt(vtt_str)\n", " # # check captions name\n", " # if lang_code.split(\"-\")[0] == \"en\":\n", " # cc_name = _get_cc_name(m[\"meta\"])\n", " # if cc_name is not None and cc_name != \"Default\":\n", " # raise ValueError(\"suspicious subtitle name\")\n", " validate_quality(annotated_lines, lang_code, duration_s=duration_s)\n", " res[f\"annotated_lines_{lang}\"] = clean_cc_lines(annotated_lines, lang_code=lang_code)\n", " res[\"success\"] = True\n", " except Exception as e:\n", " res[\"error_msg\"] = str(type(e)) + \" -- \" + str(e)\n", " return res\n", " return res" ] }, { "cell_type": "code", "execution_count": 69, "id": "4730db4a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "34.6 seconds\n" ] } ], "source": [ "out = []\n", "with open(os.path.join(DATA_DIR, \"youtube_metas.jsonl\")) as f:\n", " t0 = time.time()\n", " for n, line in enumerate(f):\n", " m = process_item(line)\n", " if m[\"success\"]:\n", " out.append(m)\n", " if len(out) >= 1000:\n", " break\n", " t1 = time.time()\n", "print(round((t1 - t0), 1), \"seconds\")" ] }, { "cell_type": "code", "execution_count": 73, "id": "156aa7ce", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "500000it [07:37, 1093.74it/s]\n", "500000it [08:01, 1037.39it/s]\n", "2049000it [31:52, 1071.21it/s]\n" ] } ], "source": [ "# single core\n", "with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\"), \"w\") as f:\n", " f.write(\"\")\n", "with open(os.path.join(PROJECT_DIR, \"failed_metas.jsonl\"), \"w\") as f:\n", " f.write(\"\")\n", "meta_fns = [\"youtube_metas.jsonl\", \"youtube_metas_p2.jsonl\", \"youtube_metas_p3.jsonl\"]\n", "# tqdm steps: 500_000, 500_000, 2_050_000\n", "for meta_fn in meta_fns:\n", " with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\"), \"a\") as fsw:\n", " with open(os.path.join(PROJECT_DIR, \"failed_metas.jsonl\"), \"a\") as few:\n", " with open(os.path.join(DATA_DIR, meta_fn)) as f:\n", " for line in tqdm.tqdm(f):\n", " m = process_item(line)\n", " if m[\"success\"]:\n", " fsw.write(json.dumps(m) + \"\\n\")\n", " else:\n", " few.write(json.dumps(m) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": 10, "id": "5773a89c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "50it [06:36, 7.93s/it]\n", "50it [07:31, 9.03s/it]\n", "205it [1:06:28, 19.46s/it]\n" ] } ], "source": [ "# multi core\n", "n_cores = 31\n", "chunksize = 10000\n", "with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\"), \"w\") as f:\n", " f.write(\"\")\n", "with open(os.path.join(PROJECT_DIR, \"failed_metas.jsonl\"), \"w\") as f:\n", " f.write(\"\")\n", "meta_fns = [\"youtube_metas.jsonl\", \"youtube_metas_p2.jsonl\", \"youtube_metas_p3.jsonl\"]\n", "# tqdm steps: 50, 50, 205\n", "for meta_fn in meta_fns:\n", " with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\"), \"a\") as fsw:\n", " with open(os.path.join(PROJECT_DIR, \"failed_metas.jsonl\"), \"a\") as few:\n", " with open(os.path.join(DATA_DIR, meta_fn)) as f:\n", " for n, lines_chunk in tqdm.tqdm(\n", " enumerate(funcy.chunks(chunksize, f.readlines()))\n", " ):\n", " with Pool(n_cores) as p:\n", " out = p.map(process_item, lines_chunk, chunksize=5)\n", " for m in out:\n", " if m[\"success\"]:\n", " fsw.write(json.dumps(m) + \"\\n\")\n", " else:\n", " few.write(json.dumps(m) + \"\\n\")\n", "# break\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "17b5cdfd", "metadata": {}, "outputs": [], "source": [ "# TODO:\n", "# this should prob not be a speaker change: Aos já inscritos - Gratidão pelas inscrições,\n", "# (3x) for lyrics?\n", "# repetitions in lyrics - don't merge" ] }, { "cell_type": "code", "execution_count": null, "id": "0d3c7276", "metadata": {}, "outputs": [], "source": [ "# !head -2 /data2/suno/data/harvest/youtube/pipeline/verified_metas.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "efd282a3", "metadata": {}, "outputs": [], "source": [ "# !head -2 /data2/suno/data/harvest/youtube/pipeline/failed_metas.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "71376ae3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f4d0744b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "ce213811", "metadata": {}, "source": [ "## Analyze (optional)" ] }, { "cell_type": "code", "execution_count": 11, "id": "5925b231", "metadata": {}, "outputs": [], "source": [ "# from suno_utils.harvest.youtube.constants_tags import KNOWN_TAGS\n", "# tags = []\n", "# with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\")) as f:\n", "# for line in f:\n", "# line = line.strip()\n", "# if len(line) == 0:\n", "# continue\n", "# m = json.loads(line)\n", "# text_lines = [s for _, s in m[\"annotated_lines\"]]\n", "# for c in set(\"\".join(text_lines)):\n", "# if ord(c) >= 983040 and ord(c) <= 1048573:\n", "# tags.append(c)\n", "\n", "# print(len(tags))\n", "# print([KNOWN_TAGS[ord(e)-983040] for e in tags[:20]])" ] }, { "cell_type": "code", "execution_count": 77, "id": "c865ed33", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "13149.8 hours\n" ] } ], "source": [ "from collections import defaultdict\n", "tot_duration_s = 0\n", "with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " tot_duration_s += m[\"duration_s\"]\n", "print(round(tot_duration_s / 60 / 60, 1), \"hours\")\n", "# 13149.8 hours" ] }, { "cell_type": "code", "execution_count": 79, "id": "434c990c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "file excluded 2745488\n", "harvest issue 144698\n", " -- punctuation information not good 24050\n", " -- casing information not good 16372\n", " -- mistakes found 13190\n", " -- a line is too long 8563\n", "too few views 5554\n", " -- no commas 3342\n", " -- timestamps too long 1193\n", " -- character rate suspicious 1143\n", "dtype: int64" ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "error_data = []\n", "with open(os.path.join(PROJECT_DIR, \"failed_metas.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " if not m[\"success\"]:\n", " error_data.append(m[\"error_msg\"])\n", "pd.Series(error_data).value_counts().head(10)" ] }, { "cell_type": "code", "execution_count": null, "id": "11fcd89a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e520a6a5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "84fdc827", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "20d31417", "metadata": {}, "source": [ "## Segmenting" ] }, { "cell_type": "code", "execution_count": 87, "id": "641a7bc0", "metadata": {}, "outputs": [], "source": [ "import uuid\n", "from suno_utils.harvest.youtube.constants.base import SUPPORTED_LANGS, NON_WHITESPACE_LANGS" ] }, { "cell_type": "code", "execution_count": 88, "id": "100a8bd4", "metadata": {}, "outputs": [], "source": [ "# TODO: if pause then add safety??\n", "# TODO: leave original codec?" ] }, { "cell_type": "code", "execution_count": 89, "id": "1d66f257", "metadata": {}, "outputs": [], "source": [ "MIN_SEGMENT_DURATION_S = 2.0\n", "MAX_SEGMENT_DURATION_S = 20.0\n", "MAX_SILENCE_DURATION_S = 5.0\n", "\n", "\n", "def create_segments(\n", " annotated_lines,\n", " lang_code,\n", " min_segment_duration_s=MIN_SEGMENT_DURATION_S,\n", " max_segment_duration_s=MAX_SEGMENT_DURATION_S,\n", " max_silence_duration_s=MAX_SILENCE_DURATION_S\n", "):\n", " if len(annotated_lines) == 0:\n", " return []\n", " merge_char = \"\" if lang_code in NON_WHITESPACE_LANGS else \" \"\n", " segments_info = []\n", " for (start_s, end_s), text in annotated_lines:\n", " if (\n", " len(segments_info) > 0 and \n", " end_s - segments_info[-1][0][0] <= max_segment_duration_s and\n", " start_s - segments_info[-1][0][1] <= max_silence_duration_s\n", " ):\n", " segments_info[-1] = (\n", " (segments_info[-1][0][0], end_s), \n", " segments_info[-1][1] + merge_char + text\n", " )\n", " else:\n", " segments_info.append(((start_s, end_s), text))\n", " segments_info = [\n", " ((start_s, end_s), text.strip(\"| \"), str(uuid.uuid4())) \n", " for (start_s, end_s), text in segments_info\n", " if end_s - start_s <= max_segment_duration_s and end_s - start_s >= min_segment_duration_s\n", " ]\n", " return segments_info" ] }, { "cell_type": "code", "execution_count": 92, "id": "7d881f87", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████| 84995/84995 [01:47<00:00, 794.08it/s]\n" ] } ], "source": [ "with open(os.path.join(PROJECT_DIR, \"segments_meta.jsonl\"), \"w\") as fw:\n", " with open(os.path.join(PROJECT_DIR, \"verified_metas.jsonl\")) as f:\n", " for line in tqdm.tqdm(f, total=84995):\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " for lang in [\"de\", \"en\"]:\n", " m = json.loads(line)\n", " segments_info = create_segments(m[f\"annotated_lines_{lang}\"], lang)\n", " if len(segments_info) == 0:\n", " continue\n", " mm = {\n", " \"id\": m[\"id\"],\n", " \"uid\": str(uuid.uuid4()),\n", " \"audio_fn\": m[\"audio_fn\"],\n", " \"segment_info\": segments_info,\n", " \"vtt_lang\": lang,\n", " }\n", " fw.write(json.dumps(mm) + \"\\n\")" ] }, { "cell_type": "code", "execution_count": 96, "id": "bcc4c372", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "9861.5 hours of en\n", "9863.8 hours of de\n" ] } ], "source": [ "import random\n", "\n", "en_total_duration_s = 0\n", "de_total_duration_s = 0\n", "with open(os.path.join(PROJECT_DIR, \"segments_meta.jsonl\")) as f:\n", " for line in tqdm.tqdm(f, total=169_983):\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " duration_s = np.sum([e[0][1] - e[0][0] for e in m[\"segment_info\"]])\n", " if m[\"vtt_lang\"] == \"en\":\n", " en_total_duration_s += duration_s\n", " else:\n", " de_total_duration_s += duration_s\n", "print(round(en_total_duration_s / 60 / 60, 1), \"hours of en\")\n", "print(round(de_total_duration_s / 60 / 60, 1), \"hours of de\")" ] }, { "cell_type": "code", "execution_count": null, "id": "e833668b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "aecaa092", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "64f4df8b", "metadata": {}, "source": [ "## Analyze segments (optional)" ] }, { "cell_type": "code", "execution_count": 84, "id": "cafab3be", "metadata": {}, "outputs": [], "source": [ "from suno_utils.audio import Audio" ] }, { "cell_type": "code", "execution_count": 85, "id": "eed5d078", "metadata": {}, "outputs": [], "source": [ "# !ls -lah /data2/suno/data/harvest/youtube/pipeline/segments_meta.jsonl" ] }, { "cell_type": "code", "execution_count": 103, "id": "f6580d4d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████| 169983/169983 [00:08<00:00, 21008.08it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "169983 samples pulled\n" ] } ], "source": [ "import random\n", "\n", "random.seed(6006)\n", "sample_segments = []\n", "n = 0\n", "with open(os.path.join(PROJECT_DIR, \"segments_meta.jsonl\")) as f:\n", " for line in tqdm.tqdm(f, total=169_983):\n", " n += 1\n", "# if n % 7 != 0:\n", "# continue\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " (start_s, end_s), text, uid = random.choice(m[\"segment_info\"])\n", " sample_segments.append((uid, m[\"audio_fn\"], m[\"vtt_lang\"], (start_s, end_s), text))\n", "# if n == 100:\n", "# break\n", "random.shuffle(sample_segments)\n", "print(len(sample_segments), \"samples pulled\")" ] }, { "cell_type": "code", "execution_count": 104, "id": "416a2651", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "de 84993\n", "en 84990\n", "Name: 2, dtype: int64" ] }, "execution_count": 104, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.DataFrame(sample_segments)[2].value_counts()" ] }, { "cell_type": "code", "execution_count": 105, "id": "07addf87", "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "1T3RHuPB_cg.webm - en - 506.005 516.816\n", "And one of the stimuluses of the brain, one of the most important systems that controls the crawling at seven, eight months, is the vision.\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "UMmhOwSlN-w.webm - en - 78.0 85.0\n", "After a failed attempt at the SRF Winter Challenge, Dani Arnold successfully completed the redpoint ascent along the \"Deep Blue Sea\" route on December 27, 2015.\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "MdnmyoLHiw4.m4a - en - 713.18 731.6\n", "in the Brown Sleeve Hotel. And the policemen desperately searched for the Brown Sleeve Hotel. And then he told them. That's the Franciscan monastery. | Brown sleeves. | Brown sleeves, yes. Brown Sleeve Hotel.\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "jIMQSHRTEMc.webm - de - 286.419 305.86\n", "Stellen Sie dann die Gläser so in den Gefrierschrank , dass sie gefroren sind und mehr Zeit und Sie haben Zeit , Spaß in Ihrem eigenen Tempo zu haben Sobald Sie dieses Rezept versuchen, ich garantieren , dass Sie nie eine Neuanschaffung machen Laden gekauft Schokoladeneis — wenn Ihnen das Rezept gefallen hat, vergessen Sie nicht\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "5lkx70Pqa68.webm - en - 556.48 575.66\n", "from the spiritual side some useful tips are: when we do our Kirtan, the \"Baba Nam Kevalam\" meditation, when we raise the hands and we imagine that our meditation is a service. So when we're singing the the \"Baba Nam Kevalam\"\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "kZwH0AXp-eg.m4a - de - 86.82 100.25\n", "Im oberen Teil der Anschluss für den Magicube Blitz\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "zhGhSzxDx2k.webm - en - 1001.909 1020.277\n", "if you understand them! And if you want to follow us to start a business, you must go to our Entrepreneurship Business School! Remember! We are a business school with the theme of entrepreneurship, but we just chose to start a business on Youtube, which is on the cusp of the storm. I have also practiced it, and I have done well in the field of knowledge. Youtuber,\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "jxP02zBRgT8.webm - de - 16.182 35.835\n", "einer Kamera vergleichen wollen, die in Full-HD-Qualität aufzeichnet, hat unser Gedächtnis genug Kapazität, um Informationen für 300 Jahre aufzuzeichnen. Ob Sie also mehrere Fremdsprachen lernen oder in mehreren Zweigen theoretischer und naturwissenschaftlicher Kenntnisse studieren möchten, hängt von Ihrer Geduld und Ausdauer ab, nicht von Ihrem Gedächtnis.\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Zcnzz2LkIsc.m4a - en - 112.189 127.909\n", "All the upholstery is at home and Iam sewing new covers. But in the near future other stuff needs to happen so that Biber becomes more seaworthy and suitable for cruising.\n", "----------\n" ] }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Yn1WFuh6Zqs.webm - de - 457.02 476.96\n", "Am einfachsten fügt ihr einen Farbverlauf mit dem Zauberstab ein. Aktiviert \"Follow Adjacent Pixel\" , um nicht alles auf einmal, sondern nur ein Panel auszuwählen.\n", "----------\n" ] } ], "source": [ "for uid, audio_fn, lang_code, (start_s, end_s), text in sample_segments[:10]:\n", " Audio.from_file(os.path.join(DATA_DIR, \"audio\", audio_fn)).play(start_s, end_s)\n", " print(audio_fn, \"-\", lang_code, \"-\", start_s, end_s)\n", " print(text)\n", " print(\"-\"*10)" ] }, { "cell_type": "code", "execution_count": null, "id": "d1d44f10", "metadata": {}, "outputs": [], "source": [ "# audio could still be ANY language" ] }, { "cell_type": "code", "execution_count": null, "id": "51523a35", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1e6c0fdf", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "25a4d774", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2549db4f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "769dcecc", "metadata": {}, "source": [ "### move segments to s3" ] }, { "cell_type": "code", "execution_count": 106, "id": "964daf0a", "metadata": {}, "outputs": [], "source": [ "# restrict to certain languages\n", "# subsample based on language prominence to get ~20k hours" ] }, { "cell_type": "code", "execution_count": 113, "id": "23d24c9c", "metadata": {}, "outputs": [], "source": [ "import tempfile\n", "import subprocess\n", "import funcy\n", "\n", "import boto3\n", "import numpy as np\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.utils.numbers import safe_round\n", "\n", "S3_BUCKET = \"suno-data\"\n", "# S3_PATH = \"datasets/harvest/youtube/dataset\"\n", "S3_PATH = \"datasets/harvest/youtube/translation_en_de\"\n", "\n", "MAX_SEGMENTS_PER_FILE = 100\n", "\n", "def write_segments(line, suppress_errors=True):\n", " try:\n", " line = line.strip()\n", " if len(line) == 0:\n", " return None\n", " m = json.loads(line)\n", " # ~150x realtime to load audio\n", " audio_fp = os.path.join(DATA_DIR, \"audio\", m[\"audio_fn\"])\n", " true_duration_s = Audio.from_file(audio_fp, sample_rate=16_000, byte_width=2).duration_s\n", " label_duration_s = Audio.get_duration_s(audio_fp, attempt_using_header=True)\n", " if np.abs(true_duration_s - label_duration_s) > 2:\n", " return None\n", " with tempfile.TemporaryDirectory() as tmp_dir:\n", " # ~80x realtime to encode entire audio, ~14x segment-by-segment\n", " # ~30x if done in single command\n", " tmp_fp = os.path.join(tmp_dir, \"tmp.opus\")\n", " # TODO: kick out files where full duration doesn't quite match (not fully scraped)\n", " out = subprocess.call(\n", " f\"ffmpeg -y -i {audio_fp} -c:a libopus -b:a 32K -ac 1 {tmp_fp}\".split(), \n", " stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n", " )\n", " convert_cmd_lines = [\n", " f\"ffmpeg -y -i {tmp_fp}\"\n", " ]\n", " segments_info = m[\"segment_info\"][:]\n", " random.shuffle(segments_info)\n", " segments_info = sorted(segments_info[:MAX_SEGMENTS_PER_FILE], key=lambda x: x[0][0])\n", " segment_metas = []\n", " for (start_s, end_s), text, segment_uid in segments_info:\n", " to_fn = f\"{segment_uid}.opus\"\n", " to_fp = os.path.join(tmp_dir, to_fn)\n", "# audio_segment = audio.get_slice(start_s, end_s)\n", "# audio_segment.to_opus(to_fp, opusenc_path=\"/bin/opusenc\")\n", " duration_s = end_s - start_s\n", " convert_cmd_lines.append(\n", " f\"-c copy -t {round(duration_s, 2)} -ss {round(start_s, 2)} {to_fp}\"\n", " )\n", " # TODO: write actual duration, not calculated\n", " segment_metas.append({\n", " \"original_id\": m[\"id\"],\n", " \"original_uid\": m[\"uid\"],\n", " \"original_offset_s\": safe_round(start_s),\n", " \"id\": segment_uid,\n", " \"lang_code\": m[\"vtt_lang\"],\n", " \"audio_filename\": to_fn,\n", " \"audio_filepath\": to_fp,\n", " \"duration_s\": safe_round(duration_s),\n", " \"text\": text,\n", " })\n", " if len(segment_metas) == 0:\n", " return None\n", " out = subprocess.call(\n", " \" \".join(convert_cmd_lines).split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,\n", " )\n", " assert(out == 0) \n", " # upload to s3\n", " s3 = boto3.resource(\"s3\")\n", " base_lang = m[\"vtt_lang\"]\n", " for m in segment_metas:\n", " to_fp = os.path.join(S3_PATH, f\"audio/{base_lang}/{m['original_uid']}/{m['audio_filename']}\")\n", " s3.Bucket(S3_BUCKET).upload_file(m[\"audio_filepath\"], to_fp)\n", " os.remove(m[\"audio_filepath\"])\n", " m[\"audio_filepath\"] = to_fp\n", " return segment_metas\n", " except Exception as e:\n", " if not suppress_errors:\n", " raise e\n", " return None" ] }, { "cell_type": "code", "execution_count": 114, "id": "61489405", "metadata": {}, "outputs": [], "source": [ "# %%time\n", "# out = write_segments(line, suppress_errors=False)\n", "# len([e for e in out if e is not None])" ] }, { "cell_type": "code", "execution_count": 131, "id": "853b8231", "metadata": {}, "outputs": [], "source": [ "# aws s3 rm --recursive s3://suno-data/datasets/harvest/youtube/translation_en_de/" ] }, { "cell_type": "code", "execution_count": null, "id": "c5c6a6ed", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 1%|▌ | 1/84 [17:05<23:38:26, 1025.38s/it]" ] } ], "source": [ "# ~7 days for entire 100k hours\n", "# 5k files - 4.9k working, 137k segments, ~630h of data -- ~50mins on 32 cores\n", "n_cores = 31\n", "chunksize = 2000\n", "random.seed(6006)\n", "with open(os.path.join(PROJECT_DIR, \"upload_segments_meta.jsonl\"), \"w\") as fw:\n", " fw.write(\"\")\n", "with open(os.path.join(PROJECT_DIR, \"segments_meta.jsonl\")) as f:\n", " for n, lines_chunk in tqdm.tqdm(\n", " enumerate(funcy.chunks(chunksize, f.readlines())), total=169_983//chunksize,\n", " ):\n", " with Pool(n_cores) as p:\n", " out = p.map(write_segments, lines_chunk, chunksize=1)\n", " out_write = [m for m in out if m is not None]\n", " with open(os.path.join(PROJECT_DIR, \"upload_segments_meta.jsonl\"), \"a\") as fw:\n", " for m in out_write:\n", " for mm in m:\n", " fw.write(json.dumps(mm) + \"\\n\")\n", "# break" ] }, { "cell_type": "code", "execution_count": 123, "id": "49ddac6c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "18733.52 hours of data\n" ] } ], "source": [ "# get durations\n", "durations_s = []\n", "with open(os.path.join(PROJECT_DIR, \"upload_segments_meta.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " durations_s.append(m[\"duration_s\"])\n", "print(round(np.sum(durations_s) / 60 / 60, 2), \"hours of data\")" ] }, { "cell_type": "code", "execution_count": 125, "id": "c0041423", "metadata": {}, "outputs": [], "source": [ "# !s3cmd put /data2/suno/data/harvest/youtube/en_de_translation/upload_segments_meta.jsonl s3://suno-data/datasets/harvest/youtube/translation_en_de/" ] }, { "cell_type": "code", "execution_count": null, "id": "751a235f", "metadata": {}, "outputs": [], "source": [ "# s4cmd ls s3://suno-data/datasets/harvest/youtube/translation_en_de/audio/en | wc -l" ] }, { "cell_type": "code", "execution_count": null, "id": "0694054d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5e90e702", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "42b86a43", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "d4a62d4c", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "ddd22c1d", "metadata": {}, "outputs": [], "source": [ "vtt_fns = os.listdir(os.path.join(DATA_DIR, \"audio\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "9ee50a7a", "metadata": {}, "outputs": [], "source": [ "l = [fn.split(\".\")[0] for fn in vtt_fns]\n", "vc = pd.Series([e for e in l]).value_counts()\n", "unique_ids = set(vc[vc == 1].index)" ] }, { "cell_type": "code", "execution_count": 125, "id": "97d9c1c4", "metadata": {}, "outputs": [], "source": [ "langs = [fn.split(\".\")[1] for fn in vtt_fns if fn.split(\".\")[0] in unique_ids]\n", "vc = pd.Series(langs).value_counts()" ] }, { "cell_type": "code", "execution_count": 126, "id": "6194bfa5", "metadata": {}, "outputs": [], "source": [ "# can we tell \"KIgOUc63dRo\" is russian?" ] }, { "cell_type": "code", "execution_count": 183, "id": "155a8129", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[youtube] rKYaV-vRc7E: Downloading webpage\n", "[download] Destination: tmp/rKYaV-vRc7E.webm\n", "[download] 100% of 1.28MiB in 00:0091MiB/s ETA 00:001\n" ] } ], "source": [ "import youtube_dl\n", "\n", "YDL_OPTS = {\n", " \"format\": \"worstaudio\",\n", " \"outtmpl\": \"tmp/%(id)s.%(ext)s\",\n", " \"writesubtitles\": True,\n", " \"allsubtitles\": True,\n", " \"subtitlesformat\": \"best\",\n", " \"socket_timeout\": 5.0,\n", "}\n", "\n", "YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n", "\n", "# url = YOUTUBE_BASE_URL + \"KIgOUc63dRo\" # russian, all sorts of stuff\n", "\n", "# url = YOUTUBE_BASE_URL + \"n9pMNI-tWzE\" # cnn, english only\n", "# len(info[\"subtitles\"].keys()) == 1\n", "\n", "url = YOUTUBE_BASE_URL + \"rKYaV-vRc7E\" # last week tonight, auto english only\n", "\n", "with youtube_dl.YoutubeDL(YDL_OPTS) as ydl:\n", " info = ydl.extract_info(url, download=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "2f59dd5f", "metadata": {}, "outputs": [], "source": [ "# TODO: next look at how many have ALL vs just a few in 'manual' category" ] }, { "cell_type": "code", "execution_count": 176, "id": "575ac7ad", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 176, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(info[\"subtitles\"].keys()) == 1" ] }, { "cell_type": "code", "execution_count": 152, "id": "fcc92b48", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "dict_keys(['af', 'ak', 'sq', 'am', 'ar', 'hy', 'as', 'ay', 'az', 'bn', 'eu', 'be', 'bho', 'bs', 'bg', 'my', 'ca', 'ceb', 'zh-Hans', 'zh-Hant', 'co', 'hr', 'cs', 'da', 'dv', 'nl', 'en', 'eo', 'et', 'ee', 'fil', 'fi', 'fr', 'gl', 'lg', 'ka', 'de', 'el', 'gn', 'gu', 'ht', 'ha', 'haw', 'iw', 'hi', 'hmn', 'hu', 'is', 'ig', 'id', 'ga', 'it', 'ja', 'jv', 'kn', 'kk', 'km', 'rw', 'ko', 'kri', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mi', 'mr', 'mn', 'ne', 'nso', 'no', 'ny', 'or', 'om', 'ps', 'fa', 'pl', 'pt', 'pa', 'qu', 'ro', 'ru', 'sm', 'sa', 'gd', 'sr', 'sn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'su', 'sw', 'sv', 'tg', 'ta', 'tt', 'te', 'th', 'ti', 'ts', 'tr', 'tk', 'uk', 'und', 'ur', 'ug', 'uz', 'vi', 'cy', 'fy', 'xh', 'yi', 'yo', 'zu'])" ] }, "execution_count": 152, "metadata": {}, "output_type": "execute_result" } ], "source": [ "info[\"automatic_captions\"].keys()" ] }, { "cell_type": "code", "execution_count": null, "id": "7b16a8eb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "2d71a354", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "55517019", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3a85b73e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8ace8d00", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9141e599", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }