{ "cells": [ { "cell_type": "code", "execution_count": 139, "id": "dbd29723", "metadata": {}, "outputs": [], "source": [ "import base64\n", "import os\n", "import re\n", "import time\n", "import random\n", "import json\n", "import tqdm\n", "import uuid\n", "import requests\n", "import string\n", "import funcy\n", "import gzip\n", "import urllib\n", "import numpy as np\n", "import multiprocessing\n", "from bs4 import BeautifulSoup\n", "from contextlib import redirect_stderr, redirect_stdout\n", "from suno_utils.utils.text import make_unique_list\n", "\n", "import pandas as pd\n", "import cloudscraper\n", "import youtube_dl\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import get_audio_properties, get_duration_s\n", "\n", "\n", "def _courtesy_sleep(avg_sleep_dur_s=0.5):\n", " time.sleep((0.5 + random.random() / 2) * avg_sleep_dur_s)\n", "\n", " \n", "class Logger():\n", " def __init__(self, filepath):\n", " self._filepath = filepath\n", " self._reset_log()\n", " \n", " def _reset_log(self):\n", " with open(self._filepath, \"w\") as f:\n", " f.write(\"\")\n", " \n", " def _add_line(self, line):\n", " with open(self._filepath, \"a\") as f:\n", " f.write(line + \"\\n\")\n", "\n", "def _parse_duration(s):\n", " if s is None:\n", " return 0\n", " parts = s.split(\":\")\n", " if len(parts) == 3:\n", " nh, nm, ns = parts\n", " elif len(parts) == 2:\n", " nm, ns = parts\n", " nh = 0\n", " else:\n", " raise ValueError(\"\")\n", " return int(nh) * 60**2 + int(nm) * 60 + int(ns)\n", " \n", "\n", "def mp_scrape(\n", " extract_f, \n", " queue_items, \n", " global_info=None,\n", " result_filepath=None, \n", " log_filepath=None, \n", " n_cores=5, \n", " chunksize=500, \n", " n_retries=3,\n", " continue_partial=False,\n", " backoff_dur_s=1.0,\n", " inner_chunksize=1, \n", " quiet=False,\n", "):\n", " if global_info is not None:\n", " _f = funcy.partial(extract_f, global_info=global_info)\n", " else:\n", " _f = extract_f\n", " if result_filepath is not None and not continue_partial:\n", " with open(result_filepath, \"w\") as f:\n", " f.write(\"\")\n", " if log_filepath is not None:\n", " logger = Logger(log_filepath)\n", " out = []\n", " n_chunks = int(np.ceil(len(queue_items) / chunksize))\n", " for n_chunk, queue_items_chunk in tqdm.tqdm(\n", " enumerate(funcy.chunks(chunksize, queue_items)), \n", " total=n_chunks,\n", " disable=quiet,\n", " ):\n", " p = multiprocessing.Pool(n_cores)\n", " t0 = time.time()\n", " remaining_items = [(idx, queue_item) for idx, queue_item in enumerate(queue_items_chunk)]\n", " out_chunk = [None] * len(queue_items_chunk)\n", " for n_retry in range(n_retries):\n", " tmp_out = p.map(_f, [queue_item for _, queue_item in remaining_items], chunksize=inner_chunksize)\n", " tmp_remaining_items = []\n", " for (idx, queue_item), tmp_out_item in zip(remaining_items, tmp_out):\n", " out_chunk[idx] = tmp_out_item\n", " if isinstance(tmp_out_item, dict) and tmp_out_item.get(\"success\") == False:\n", " tmp_remaining_items.append((idx, queue_item))\n", " continue\n", " remaining_items = tmp_remaining_items\n", " if len(remaining_items) == 0:\n", " break\n", " if n_retry < n_retries - 1:\n", " logger._add_line(f\" retrying for {len(remaining_items)}/{len(queue_items_chunk)} items\")\n", " else:\n", " logger._add_line(f\" failed on {len(remaining_items)}/{len(queue_items_chunk)} items\")\n", " _courtesy_sleep(avg_sleep_dur_s=backoff_dur_s)\n", " if result_filepath is not None:\n", " with open(result_filepath, \"a\") as f:\n", " for e in out_chunk:\n", " f.write(json.dumps(e) + \"\\n\")\n", " else:\n", " out.extend(out_chunk)\n", " td = time.time() - t0\n", " if log_filepath is not None:\n", " logger._add_line(f\"{n_chunk+1}/{n_chunks} - last step took {round(td / 60, 1)} mins\")\n", " p.close()\n", " p.join()\n", " logger._add_line(f\"done!\")\n", " if result_filepath is not None:\n", " return None\n", " return out\n", "\n", "US_PROXY = (\n", " \"http://brd-customer-hl_a8c79c59-zone-us_pool:\" +\n", " \"cdojde57liiz@zproxy.lum-superproxy.io:22225\"\n", ")\n", "\n", "DATA_DIR = \"/data/suno/data/harvest/genius\"" ] }, { "cell_type": "markdown", "id": "7af110f9", "metadata": {}, "source": [ "## Prep data items" ] }, { "cell_type": "code", "execution_count": 2, "id": "6041cb74", "metadata": {}, "outputs": [], "source": [ "# song_metas = []\n", "# with open(os.path.join(DATA_DIR, \"artist_songs.jsonl\")) as f:\n", "# for line in f:\n", "# line = line.strip()\n", "# if len(line) == 0:\n", "# continue\n", "# m = json.loads(line)\n", "# if m is None:\n", "# continue\n", "# # if m[\"meta\"][\"youtube_url\"] is None:\n", "# # continue\n", "# song_metas.append(m)\n", "# if len(song_metas) == 12:\n", "# break" ] }, { "cell_type": "code", "execution_count": 3, "id": "ddc225b5", "metadata": {}, "outputs": [], "source": [ "# lyrics = []\n", "# with open(os.path.join(DATA_DIR, \"song_details.jsonl\")) as f:\n", "# for line in f:\n", "# line = line.strip()\n", "# if len(line) == 0:\n", "# continue\n", "# m = json.loads(line)\n", "# if m is None:\n", "# continue\n", "# if m[\"meta\"][\"youtube_url\"] is None:\n", "# continue\n", "# lyrics.append(m)\n", "# if len(lyrics) == 12:\n", "# break" ] }, { "cell_type": "code", "execution_count": 19, "id": "2c07ff30", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "73943 items loaded.\n" ] } ], "source": [ "base_metas = {}\n", "with open(os.path.join(DATA_DIR, \"de_project\", \"base_metas.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " if m is None:\n", " continue\n", " base_metas[m[\"song_slug\"]] = m\n", "# if len(base_metas) == 12:\n", "# break\n", "print(len(base_metas), \"items loaded.\")" ] }, { "cell_type": "code", "execution_count": 120, "id": "5a578a5b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "39731 items loaded.\n" ] } ], "source": [ "seen_slugs = set()\n", "song_metas = []\n", "with open(os.path.join(DATA_DIR, \"de_project\", \"new_metas.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " if m is None:\n", " continue\n", " if m[\"song_slug\"] in seen_slugs:\n", " continue\n", " youtube_url = base_metas[m[\"song_slug\"]][\"meta\"][\"youtube_url\"]\n", " if youtube_url is None:\n", " continue\n", " # get tags & pageviews\n", " details_key = None\n", " for k, v in m[\"meta\"][\"songPage\"].items():\n", " if isinstance(v, list) and any([isinstance(vv, dict) and vv.get(\"name\", \"\") == \"song_id\" for vv in v]):\n", " details_key = k\n", " break\n", " if details_key is None:\n", " continue\n", " info = m[\"meta\"][\"songPage\"][details_key]\n", " page_views = None\n", " for d in info:\n", " if \"name\" in d and d[\"name\"] == \"pageviews\":\n", " page_views = int(d[\"values\"][0])\n", " break\n", " tags = None\n", " for d in info:\n", " if \"name\" in d and d[\"name\"] == \"tag_id\":\n", " tags = [int(e) for e in d[\"values\"]]\n", " break\n", " if tags is None or not any([n == 17 for n in tags]):\n", " continue\n", " # get more song details\n", " song_id = str(m[\"meta\"][\"songPage\"][\"song\"])\n", " song_details = m[\"meta\"][\"entities\"][\"songs\"][song_id]\n", " lang_lyrics = None\n", " for mm in song_details[\"trackingData\"]:\n", " if mm[\"key\"] == \"Lyrics Language\":\n", " lang_lyrics = mm[\"value\"]\n", " break\n", " youtube_start = song_details[\"youtubeStart\"]\n", " if youtube_start is None or len(youtube_start) == 0:\n", " youtube_start = \"0\"\n", " song_metas.append({\n", " \"slug\": m[\"song_slug\"],\n", " \"id\": song_id,\n", " \"youtube_url\": song_details[\"youtubeUrl\"],\n", " \"youtube_start\": youtube_start,\n", " \"lang\": song_details[\"language\"],\n", " \"lang_lyrics\": lang_lyrics,\n", " \"views\": page_views,\n", " \"tags\": tags,\n", " \"lyrics\": base_metas[m[\"song_slug\"]][\"meta\"][\"lyrics\"],\n", "# \"lyrics_html\": m[\"meta\"][\"songPage\"]['lyricsData'][\"body\"][\"html\"],\n", " })\n", " seen_slugs.add(m[\"song_slug\"])\n", "# if len(song_metas) == 12:\n", "# break\n", "print(len(song_metas), \"items loaded.\")" ] }, { "cell_type": "code", "execution_count": 121, "id": "6b832c54", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(DATA_DIR, \"de_project\", \"song_metas.jsonl\"), \"w\") as f:\n", " for m in song_metas:\n", " f.write(json.dumps(m) + \"\\n\")" ] }, { "cell_type": "markdown", "id": "e20fa126", "metadata": {}, "source": [ "## Download from youtube" ] }, { "cell_type": "code", "execution_count": 131, "id": "72fd1cb0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "38753 items loaded.\n" ] } ], "source": [ "song_metas = []\n", "with open(os.path.join(DATA_DIR, \"de_project\", \"song_metas.jsonl\")) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " song_metas.append(json.loads(line))\n", "youtube_ids = [\n", " youtube_id\n", " for m in song_metas \n", " if (\n", " m[\"lang\"] == \"de\" and \n", " m[\"lang_lyrics\"] == \"de\" and \n", " m[\"youtube_start\"] == \"0\" and \n", " len((youtube_id := m[\"youtube_url\"].split(\"=\")[-1])) == 11\n", " )\n", "]\n", "print(len(youtube_ids), \"items loaded.\")\n", "# 38753 items loaded." ] }, { "cell_type": "code", "execution_count": 175, "id": "3138eecd", "metadata": {}, "outputs": [], "source": [ "YDL_OPTS = {\n", " \"format\": \"worstaudio\",\n", " \"outtmpl\": os.path.join(DATA_DIR, \"de_project\", \"videos\", \"%(id)s.%(ext)s\"),\n", " \"writesubtitles\": True,\n", " \"allsubtitles\": True,\n", " \"subtitlesformat\": \"best\",\n", " \"proxy\": US_PROXY,\n", "}\n", "\n", "YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n", "\n", "def _resolve_youtube(youtube_slug):\n", " url = YOUTUBE_BASE_URL + youtube_slug\n", " out_data = {\n", " \"id\": youtube_slug,\n", " }\n", " t0 = time.time()\n", " try:\n", " with redirect_stderr(open(os.devnull, \"w\")):\n", " with redirect_stdout(open(os.devnull, \"w\")):\n", " with youtube_dl.YoutubeDL(YDL_OPTS) as ydl:\n", " info = ydl.extract_info(url, download=True)\n", " out_data[\"success\"] = True\n", " out_data[\"meta\"] = info\n", " out_data[\"audio_filename\"] = f\"{info['id']}.{info['ext']}\"\n", " except Exception as e:\n", " out_data[\"success\"] = False\n", " out_data[\"fail_type\"] = str(type(e))\n", " out_data[\"fail_message\"] = str(e)\n", " t1 = time.time()\n", " out_data[\"runtime_s\"] = round(t1 - t0, 1)\n", " return out_data" ] }, { "cell_type": "code", "execution_count": 174, "id": "b2e62ebb", "metadata": {}, "outputs": [], "source": [ "# out = _resolve_youtube_video(youtube_ids[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "87d9217d", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 51%|█████████████████████████████████████████████████████████████▌ | 20/39 [1:04:14<1:00:43, 191.78s/it]" ] } ], "source": [ "# ~1h for 4k videos, ~200 hours of audio (5 cores)\n", "_ = mp_scrape(\n", " _resolve_youtube, \n", " youtube_ids, \n", " chunksize=1000,\n", " n_cores=50,\n", " n_retries=2,\n", " result_filepath=os.path.join(DATA_DIR, \"de_project\", \"youtube_metas.jsonl\"),\n", " log_filepath=os.path.join(DATA_DIR, \"logs\", \"audio_harvest.log\"),\n", ")" ] }, { "cell_type": "code", "execution_count": 181, "id": "af2ce70e", "metadata": {}, "outputs": [], "source": [ "# !ls /data/suno/data/harvest/genius/de_project/videos | wc -l" ] }, { "cell_type": "code", "execution_count": null, "id": "0bb03dae", "metadata": {}, "outputs": [], "source": [ "# TODO: move everything to s3 and then to lambda\n", "# TODO: probably want to filter on views??\n", "# TODO: parse lyrics\n", "# TODO: look at matches and try to filter stuff to high quality" ] }, { "cell_type": "code", "execution_count": null, "id": "486eb5cd", "metadata": {}, "outputs": [], "source": [ "# s4cmd put /data/suno/data/harvest/genius/de_project/song_metas.jsonl s3://suno-data/harvest/rap_de/song_metas.jsonl\n", "# s4cmd put /data/suno/data/harvest/genius/de_project/youtube_metas.jsonl s3://suno-data/harvest/rap_de/youtube_metas.jsonl\n", "# s4cmd dsync -r -n /data/suno/data/harvest/genius/de_project/videos s3://suno-data/harvest/rap_de/videos" ] }, { "cell_type": "code", "execution_count": null, "id": "bea4dbfe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "968a5929", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "61ce0c3d", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": 8, "id": "cfce4e04", "metadata": {}, "outputs": [], "source": [ "def _resolve_audio(youtube_id, global_info):\n", " audio_dir = global_info[\"audio_dir\"]\n", " try:\n", " youtube_url = f\"http://www.youtube.com/watch?v={youtube_id}\"\n", " to_filepath = os.path.join(audio_dir, f\"{youtube_id}.webm\")\n", " if not os.path.exists(to_filepath):\n", " ydl_opts = {\n", " \"outtmpl\": to_filepath,\n", " \"format\": \"worstaudio/worst\",\n", " }\n", " with redirect_stderr(open(os.devnull, \"w\")):\n", " with redirect_stdout(open(os.devnull, \"w\")):\n", " with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " out = ydl.download([youtube_url])\n", " assert(out == 0)\n", " except Exception as e:\n", " return {\n", " \"id\": youtube_id,\n", " \"success\": False,\n", " \"fail_type\": str(type(e)),\n", " \"fail_message\": str(e),\n", " }\n", " _courtesy_sleep(avg_sleep_dur_s=1.0)\n", " return {\n", " \"id\": youtube_id,\n", " \"success\": True,\n", " }" ] }, { "cell_type": "code", "execution_count": 9, "id": "6b6ffc49", "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'youtube_ids' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/tmp/ipykernel_11795/436570717.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m out = mp_scrape(\n\u001b[1;32m 3\u001b[0m \u001b[0m_resolve_audio\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0myoutube_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m200\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mn_cores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mNameError\u001b[0m: name 'youtube_ids' is not defined" ] } ], "source": [ "# ~1h for 4k videos, ~200 hours of audio (5 cores)\n", "out = mp_scrape(\n", " _resolve_audio, \n", " youtube_ids, \n", " chunksize=200,\n", " n_cores=5,\n", " n_retries=1,\n", " global_info={\n", " \"audio_dir\": GENIUS_AUDIO_DIR,\n", " },\n", " log_filepath=os.path.join(GENIUS_LOG_DIR, \"audio_harvest.log\"),\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "6b47e960", "metadata": {}, "outputs": [], "source": [ "# TODO: check number of failed ones by type" ] }, { "cell_type": "code", "execution_count": null, "id": "a1e75fa8", "metadata": {}, "outputs": [], "source": [ "filenames = [fn for fn in os.listdir(GENIUS_AUDIO_DIR) if fn.endswith(\".webm\")]\n", "print(len(filenames), \"audio files harvested.\")\n", "tmp_duration_s = 0\n", "tmp_filesize = 0\n", "for fn in filenames[:50]:\n", " fp = os.path.join(GENIUS_AUDIO_DIR, fn)\n", " tmp_filesize += os.path.getsize(fp)\n", " tmp_duration_s += get_audio_properties(fp)[\"duration_s\"]\n", "avg_duration_s = tmp_duration_s / tmp_filesize \n", "duration_s = np.sum([os.path.getsize(os.path.join(GENIUS_AUDIO_DIR, fn)) * avg_duration_s for fn in filenames])\n", "print(\"~{} hours of audio.\".format(round(duration_s / 60 / 60, 1)))" ] }, { "cell_type": "code", "execution_count": null, "id": "0bf60150", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f494f974", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e7e77d0d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "039e2bf8", "metadata": {}, "source": [ "- new machine for youtube harvesting\n", "test 100 core and see if any slowdowns (save time per item). do we need to kick out mp4s? is worstaudio/worst ok?\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b7e316cd", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import shutil\n", "import time\n", "import youtube_dl\n", "import multiprocessing\n", "\n", "US_PROXY = (\n", " \"http://brd-customer-hl_a8c79c59-zone-us_pool:\" +\n", " \"cdojde57liiz@zproxy.lum-superproxy.io:22225\"\n", ")\n", "\n", "YOUTUBE_URLS = [\n", " 'http://www.youtube.com/watch?v=OzrkFekIbxE',\n", " 'http://www.youtube.com/watch?v=PkMdMl_Kv8U',\n", " 'http://www.youtube.com/watch?v=lufaOsruNoY', # mp4 (TODO: fix later, sometimes stalls)\n", " 'http://www.youtube.com/watch?v=bVli8wC6soM',\n", " 'http://www.youtube.com/watch?v=4M-QEQheWvU',\n", " 'http://www.youtube.com/watch?v=-SVPi9j_vRE', # mp4 (TODO: fix later, sometimes stalls)\n", " 'http://www.youtube.com/watch?v=Ej4f12q7FRw',\n", "# 'http://www.youtube.com/watch?v=Xz_HpdEb_KE',\n", "# 'http://www.youtube.com/watch?v=-6hhv9vV9jk',\n", "# 'http://www.youtube.com/watch?v=Sht74F_RAI8',\n", "# 'http://www.youtube.com/watch?v=Of-w8nwBWTM',\n", "]\n", "\n", "# youtube_url = \"http://www.youtube.com/watch?v=oktMBFQNeQM\" # no subtitles\n", "# youtube_url = \"http://www.youtube.com/watch?v=mBxNwuxFYxo\" # automated and en-US\n", "# youtube_url = \"http://www.youtube.com/watch?v=zuBEwXo69nA\" # automated and en\n", "# youtube_url = \"http://www.youtube.com/watch?v=Fpu5a0Bl8eY\" # en/de\n", "\n", "# DATA_DIR = \"/data/suno/data/harvest/youtube_lg\"\n", "DATA_DIR = \"/data/suno/data/harvest/test\"\n", "\n", "# TODO: we want to list formats to handle mp4??" ] }, { "cell_type": "code", "execution_count": null, "id": "c37d15fb", "metadata": {}, "outputs": [], "source": [ "shutil.rmtree(DATA_DIR, ignore_errors=True)\n", "os.makedirs(DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "e4fb834e", "metadata": {}, "outputs": [], "source": [ "url = []\n", "url = []\n", "\n", "ydl_opts = {\n", " \"format\": \"worstaudio\",\n", " \"outtmpl\": os.path.join(DATA_DIR, \"%(id)s.%(ext)s\"),\n", " \"writesubtitles\": True,\n", " \"allsubtitles\": True,\n", " \"subtitlesformat\": \"best\",\n", " \"proxy\": US_PROXY,\n", "}\n", "t0 = time.time()\n", "with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " info = ydl.extract_info(YOUTUBE_URLS[5], download=False)\n", "# error_code = ydl.download([url, YOUTUBE_URLS[0]])\n", "# assert(error_code == 0)\n", "t1 = time.time()\n", "print(round(t1 - t0, 1), \"seconds\")\n", "filename = f\"{info['id']}.{info['ext']}\"\n", "print(filename)" ] }, { "cell_type": "code", "execution_count": null, "id": "72ded7a7", "metadata": {}, "outputs": [], "source": [ "!ls /data/suno/data/harvest/test" ] }, { "cell_type": "code", "execution_count": null, "id": "ef20e260", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "808849d9", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "332b2049", "metadata": {}, "outputs": [], "source": [ "-output filename?\n", "meta = ydl.extract_info('https://www.youtube.com/watch?v=O4xNJsjtN6E', download=False)\n", "print (\"{0}.{1}x{2}.{3}.{4}\".format((meta['title']), (meta['width']), (meta['height']), (meta['id']), (meta['ext'])))\n", "\n", " def my_hook(d):\n", " if d['status'] == 'finished':\n", " print(d['filename'])\n", "\n", " ydl_opts = {\n", " 'format': 'bestaudio',\n", " 'progress_hooks': [my_hook],\n", " }\n", " with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " ydl.download(['https://www.youtube.com/watch?v=BaW_jenozKc'])" ] }, { "cell_type": "code", "execution_count": null, "id": "5181ebe2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c0364d2b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "defbc557", "metadata": {}, "outputs": [], "source": [ "shutil.rmtree(DATA_DIR, ignore_errors=True)\n", "os.makedirs(DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "510934ae", "metadata": {}, "outputs": [], "source": [ "import youtube_dl\n", "\n", "ydl_opts = {\n", "# \"skip_download\": True,\n", " \"format\": \"worstaudio\",\n", " \"outtmpl\": os.path.join(DATA_DIR, \"%(id)s.%(ext)s\"),\n", "# \"proxy\": US_PROXY,\n", "}\n", "t0 = time.time()\n", "with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " ydl.download(YOUTUBE_URLS)\n", "t1 = time.time()\n", "print(round(t1 - t0, 1), \"seconds\")\n", "s = 0\n", "for fn in os.listdir(DATA_DIR):\n", " s += os.path.getsize(os.path.join(DATA_DIR, fn))\n", "assert(s == 7514746)" ] }, { "cell_type": "code", "execution_count": null, "id": "02dcbb41", "metadata": {}, "outputs": [], "source": [ "shutil.rmtree(DATA_DIR, ignore_errors=True)\n", "os.makedirs(DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "4314d997", "metadata": {}, "outputs": [], "source": [ "ydl_opts = {\n", "# \"skip_download\": True,\n", " \"format\": \"worstaudio\",\n", " \"outtmpl\": os.path.join(DATA_DIR, \"%(id)s.%(ext)s\"),\n", "# \"proxy\": US_PROXY,\n", "}\n", "def foo(url):\n", " with youtube_dl.YoutubeDL(ydl_opts) as ydl:\n", " ydl.download([url])\n", " \n", "p = multiprocessing.Pool(5)\n", "t0 = time.time()\n", "p.map(foo, YOUTUBE_URLS)\n", "t1 = time.time()\n", "print(round(t1 - t0, 1), \"seconds\")\n", "p.close()\n", "p.join()\n", "s = 0\n", "for fn in os.listdir(DATA_DIR):\n", " s += os.path.getsize(os.path.join(DATA_DIR, fn))\n", "assert(s == 7514746)" ] }, { "cell_type": "code", "execution_count": null, "id": "e514e0a7", "metadata": {}, "outputs": [], "source": [ "shutil.rmtree(DATA_DIR, ignore_errors=True)\n", "os.makedirs(DATA_DIR, exist_ok=True)" ] }, { "cell_type": "markdown", "id": "6d3fe781", "metadata": {}, "source": [ "## try other package" ] }, { "cell_type": "code", "execution_count": null, "id": "da179da1", "metadata": {}, "outputs": [], "source": [ "import yt_dlp\n", "\n", "ydl_opts = {\n", " 'format': 'worstaudio',\n", " \"outtmpl\": os.path.join(DATA_DIR, \"%(id)s.%(ext)s\"),\n", "# 'postprocessors': [{ # Extract audio using ffmpeg\n", "# 'key': 'FFmpegExtractAudio',\n", "# 'preferredcodec': 'm4a',\n", "# }],\n", "}\n", "\n", "t0 = time.time()\n", "for n in range(5):\n", " with yt_dlp.YoutubeDL(ydl_opts) as ydl:\n", " error_code = ydl.download([YOUTUBE_URLS[n]])\n", "t1 = time.time()\n", "print(round(t1 - t0, 1), \"seconds\")" ] }, { "cell_type": "code", "execution_count": null, "id": "e072ef6b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b3572b43", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "cdbc48a5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "250899c9", "metadata": {}, "outputs": [], "source": [ "# # logging\n", "# import yt_dlp\n", "\n", "# INFO_FILE = 'path/to/video.info.json'\n", "\n", "# with yt_dlp.YoutubeDL() as ydl:\n", "# error_code = ydl.download_with_info_file(INFO_FILE)\n", "\n", "# print('Some videos failed to download' if error_code\n", "# else 'All videos successfully downloaded')" ] }, { "cell_type": "code", "execution_count": null, "id": "d3530a5a", "metadata": {}, "outputs": [], "source": [ "# see if we can get info on language\n", "import json\n", "import yt_dlp\n", "\n", "url = \"https://www.youtube.com/watch?v=kDA-phXyYH8\" # english\n", "# url = \"https://www.youtube.com/watch?v=GOg1mX48tXU\" # german\n", "\n", "ydl_opts = {}\n", "with yt_dlp.YoutubeDL(ydl_opts) as ydl:\n", " info = ydl.extract_info(url, download=False)\n", " json_info = ydl.sanitize_info(info)\n", " print([k for k in json_info[\"automatic_captions\"].keys() if k.endswith(\"orig\")])" ] }, { "cell_type": "code", "execution_count": null, "id": "536e5d9b", "metadata": {}, "outputs": [], "source": [ "json_info" ] }, { "cell_type": "code", "execution_count": null, "id": "2cd7a810", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ba434bb0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5f8d320f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5f567c09", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }