{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "204eb465", "metadata": {}, "outputs": [], "source": [ "import base64\n", "import os\n", "import re\n", "import time\n", "import random\n", "import json\n", "import tqdm\n", "import uuid\n", "import requests\n", "import string\n", "import funcy\n", "import gzip\n", "import urllib\n", "import numpy as np\n", "import multiprocessing\n", "from bs4 import BeautifulSoup\n", "from contextlib import redirect_stderr\n", "from suno_utils.utils.text import make_unique_list\n", "\n", "import pandas as pd\n", "import cloudscraper\n", "import youtube_dl\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.audio.conversion import get_audio_properties\n", "\n", "\n", "def _courtesy_sleep(avg_sleep_dur_s=0.5):\n", " time.sleep((0.5 + random.random() / 2) * avg_sleep_dur_s)\n", "\n", " \n", "class Logger():\n", " def __init__(self, filepath):\n", " self._filepath = filepath\n", " self._reset_log()\n", " \n", " def _reset_log(self):\n", " with open(self._filepath, \"w\") as f:\n", " f.write(\"\")\n", " \n", " def _add_line(self, line):\n", " with open(self._filepath, \"a\") as f:\n", " f.write(line + \"\\n\")\n", "\n", "def _parse_duration(s):\n", " if s is None:\n", " return 0\n", " parts = s.split(\":\")\n", " if len(parts) == 3:\n", " nh, nm, ns = parts\n", " elif len(parts) == 2:\n", " nm, ns = parts\n", " nh = 0\n", " else:\n", " raise ValueError(\"\")\n", " return int(nh) * 60**2 + int(nm) * 60 + int(ns)\n", " \n", "\n", "def mp_scrape(\n", " extract_f, \n", " queue_items, \n", " global_info=None,\n", " result_filepath=None, \n", " log_filepath=None, \n", " n_cores=5, \n", " chunksize=500, \n", " n_retries=3,\n", " continue_partial=False,\n", " backoff_dur_s=1.0,\n", " inner_chunksize=1, \n", " quiet=False,\n", "):\n", " if global_info is not None:\n", " _f = funcy.partial(extract_f, global_info=global_info)\n", " else:\n", " _f = extract_f\n", " if result_filepath is not None and not continue_partial:\n", " with open(result_filepath, \"w\") as f:\n", " f.write(\"\")\n", " if log_filepath is not None:\n", " logger = Logger(log_filepath)\n", " out = []\n", " n_chunks = int(np.ceil(len(queue_items) / chunksize))\n", " for n_chunk, queue_items_chunk in tqdm.tqdm(\n", " enumerate(funcy.chunks(chunksize, queue_items)), \n", " total=n_chunks,\n", " disable=quiet,\n", " ):\n", " p = multiprocessing.Pool(n_cores)\n", " t0 = time.time()\n", " remaining_items = [(idx, queue_item) for idx, queue_item in enumerate(queue_items_chunk)]\n", " out_chunk = [None] * len(queue_items_chunk)\n", " for n_retry in range(n_retries):\n", " tmp_out = p.map(_f, [queue_item for _, queue_item in remaining_items], chunksize=inner_chunksize)\n", " tmp_remaining_items = []\n", " for (idx, queue_item), tmp_out_item in zip(remaining_items, tmp_out):\n", " out_chunk[idx] = tmp_out_item\n", " if isinstance(tmp_out_item, dict) and tmp_out_item.get(\"success\") == False:\n", " tmp_remaining_items.append((idx, queue_item))\n", " continue\n", " remaining_items = tmp_remaining_items\n", " if len(remaining_items) == 0:\n", " break\n", " if n_retry < n_retries - 1:\n", " logger._add_line(f\" retrying for {len(remaining_items)}/{len(queue_items_chunk)} items\")\n", " else:\n", " logger._add_line(f\" failed on {len(remaining_items)}/{len(queue_items_chunk)} items\")\n", " _courtesy_sleep(avg_sleep_dur_s=backoff_dur_s)\n", " if result_filepath is not None:\n", " with open(result_filepath, \"a\") as f:\n", " for e in out_chunk:\n", " f.write(json.dumps(e) + \"\\n\")\n", " else:\n", " out.extend(out_chunk)\n", " td = time.time() - t0\n", " if log_filepath is not None:\n", " logger._add_line(f\"{n_chunk+1}/{n_chunks} - last step took {round(td / 60, 1)} mins\")\n", " p.close()\n", " p.join()\n", " logger._add_line(f\"done!\")\n", " if result_filepath is not None:\n", " return None\n", " return out\n", "\n", "US_SERVER_IPS = [\n", " \"91.92.218.36\",\n", " \"162.43.235.202\",\n", " \"213.188.91.125\",\n", " \"206.204.41.115\",\n", " \"206.204.12.199\",\n", "]\n", "\n", "GLOBAL_SERVER_IPS = {\n", " 'AE': '92.51.55.104',\n", " 'AL': '45.134.106.197',\n", " 'AM': '45.143.107.144',\n", " 'AR': '5.182.127.179',\n", " 'AT': '154.37.69.108',\n", " 'AU': '94.139.44.170',\n", " 'AZ': '103.119.111.225',\n", " 'BA': '185.253.26.25',\n", " 'BD': '43.228.239.130',\n", " 'BE': '161.123.22.105',\n", " 'BG': '45.84.80.151',\n", " 'BO': '45.148.107.78',\n", " 'BR': '94.139.232.228',\n", " 'BY': '66.118.42.157',\n", " 'CA': '45.61.159.147',\n", " 'CH': '204.217.149.11',\n", " 'CL': '185.15.179.92',\n", " 'CN': '45.138.235.96',\n", " 'CO': '89.249.57.90',\n", " 'CR': '103.225.130.91',\n", " 'CY': '45.130.120.128',\n", " 'DE': '45.67.1.152',\n", " 'DK': '165.140.198.219',\n", " 'DO': '93.114.10.129',\n", " 'EC': '45.11.235.50',\n", " 'EE': '45.134.115.3',\n", " 'EG': '45.154.123.5',\n", " 'ES': '194.5.225.46',\n", " 'FK': '209.242.219.17',\n", " 'FR': '209.20.186.153',\n", " 'GB': '94.176.133.86',\n", " 'GE': '178.171.108.21',\n", " 'GR': '103.187.242.5',\n", " 'GS': '209.242.220.103',\n", " 'HK': '86.62.61.34',\n", " 'HR': '103.225.131.81',\n", " 'HU': '45.88.100.166',\n", " 'ID': '93.114.5.139',\n", " 'IE': '212.80.221.237',\n", " 'IL': '158.46.181.223',\n", " 'IM': '45.130.122.137',\n", " 'IN': '119.13.226.25',\n", " 'IQ': '194.53.68.136',\n", " 'IS': '45.130.121.44',\n", " 'IT': '206.232.41.74',\n", " 'JM': '93.113.53.126',\n", " 'JO': '93.114.13.9',\n", " 'JP': '95.175.80.37',\n", " 'KG': '193.106.96.50',\n", " 'KH': '93.114.14.118',\n", " 'KR': '178.171.98.100',\n", " 'KW': '45.158.215.245',\n", " 'KZ': '45.92.86.2',\n", " 'LA': '5.253.185.135',\n", " 'LK': '109.70.67.88',\n", " 'LU': '207.230.124.37',\n", " 'MA': '45.95.130.246',\n", " 'MD': '91.132.126.84',\n", " 'MK': '85.208.148.127',\n", " 'MS': '209.242.222.40',\n", " 'MX': '193.228.72.10',\n", " 'NL': '178.171.113.83',\n", " 'NO': '95.214.102.82',\n", " 'NZ': '69.85.88.142',\n", " 'OM': '45.130.123.5',\n", " 'PA': '92.240.206.100',\n", " 'PE': '193.111.187.187',\n", " 'PK': '103.241.52.55',\n", " 'PL': '185.251.249.179',\n", " 'PT': '91.245.238.181',\n", " 'RO': '103.14.105.177',\n", " 'RS': '85.208.149.12',\n", " 'RU': '176.53.216.74',\n", " 'SA': '194.187.36.190',\n", " 'SE': '212.80.202.240',\n", " 'SG': '104.251.88.228',\n", " 'SI': '45.150.20.114',\n", " 'SK': '45.143.174.247',\n", " 'SL': '103.225.128.37',\n", " 'TH': '158.46.171.241',\n", " 'TJ': '158.46.185.73',\n", " 'TM': '178.171.66.248',\n", " 'TN': '103.225.129.214',\n", " 'TR': '109.198.46.59',\n", " 'TW': '178.171.121.148',\n", " 'UA': '85.31.52.111',\n", " 'US': '94.176.84.186',\n", " 'VN': '69.85.92.6',\n", " 'ZA': '93.113.125.219',\n", "}\n", "\n", "with open(\"/home/georg/.secrets/secrets.json\") as f:\n", " secrets = json.load(f)\n", "\n", "US_PROXY_USER = \"lum-customer-hl_a8c79c59-zone-us_pool-route_err-block\"\n", "US_PROXY_PW = secrets[\"us_proxy_pw\"]\n", "US_PROXY_PORT = 22225\n", "\n", "def get_us_proxy_url(idx_offset=None):\n", " if idx_offset is not None:\n", " server_idx = idx_offset % len(US_SERVER_IPS)\n", " else:\n", " server_idx = random.randint(0, len(US_SERVER_IPS) - 1)\n", " ip_addr = US_SERVER_IPS[server_idx]\n", " super_proxy_url = (\n", " f\"http://{US_PROXY_USER}-ip-{ip_addr}-country-us:{US_PROXY_PW}\"\n", " f\"@zproxy.lum-superproxy.io:{US_PROXY_PORT}\"\n", " )\n", " return super_proxy_url\n", "\n", "GLOBAL_PROXY_USER = \"lum-customer-hl_a8c79c59-zone-data_center-route_err-block\"\n", "GLOBAL_PROXY_PW = secrets[\"global_proxy_pw\"]\n", "GLOBAL_PROXY_PORT = 22225\n", "\n", "def get_global_proxy_url(country_code=None):\n", " if country_code is not None:\n", " country_code = country_code.upper()\n", " if country_code is None:\n", " country_code = random.choice(list(GLOBAL_SERVER_IPS.keys()))\n", " ip_addr = GLOBAL_SERVER_IPS[country_code]\n", " super_proxy_url = (\n", " f\"http://{GLOBAL_PROXY_USER}-ip-{ip_addr}-country-{country_code.lower()}:{GLOBAL_PROXY_PW}\"\n", " f\"@zproxy.lum-superproxy.io:{GLOBAL_PROXY_PORT}\"\n", " )\n", " return super_proxy_url" ] }, { "cell_type": "code", "execution_count": 2, "id": "ffac48cf", "metadata": {}, "outputs": [], "source": [ "# DATA_DIR = \"/data/suno/data/harvest/youtube_small\"\n", "DATA_DIR = \"/data/suno/data/harvest/youtube_lg\"\n", "SUBTITLES_DIR = os.path.join(DATA_DIR, \"subtitles\")\n", "AUDIO_DIR = os.path.join(DATA_DIR, \"audio\")\n", "LOGS_DIR = os.path.join(DATA_DIR, \"logs\")\n", "\n", "os.makedirs(DATA_DIR, exist_ok=True)\n", "os.makedirs(SUBTITLES_DIR, exist_ok=True)\n", "os.makedirs(AUDIO_DIR, exist_ok=True)\n", "os.makedirs(LOGS_DIR, exist_ok=True)" ] }, { "cell_type": "markdown", "id": "b64248e1", "metadata": {}, "source": [ "### Use web search" ] }, { "cell_type": "code", "execution_count": 3, "id": "f680d8d1", "metadata": {}, "outputs": [], "source": [ "GIGA_DOMAINS = [\n", " \"Arts\", \"Business\", \"Education\", \"Autos and Vehicles\", \"Comedy\", \"Crime\", \"Entertainment\", \n", " \"Film and Animation\", \"Gaming\", \"Health and Fitness\", \"History\", \"Howto and Style\", \n", " \"Kids and Family, Leisure\", \"Music\", \"News and Politics\", \"Nonprofits and Activism\", \n", " \"People and Blogs\", \"Pets and Animals\", \"Religion and Spirituality\", \"Science and Technology\", \n", " \"Society and Culture\", \"Sports\", \"Travel and Events\",\n", "]\n", "\n", "COUNTRY_NAMES = [\n", " 'Algeria', 'Argentina', 'Australia', 'Austria', 'Azerbaijan', 'Bahrain', 'Bangladesh', 'Belarus', \n", " 'Belgium', 'Bolivia', 'Bosnia and Herzegovina', 'Brazil', 'Bulgaria', 'Cambodia', 'Canada', 'Chile', \n", " 'Colombia', 'Costa Rica', 'Croatia', 'Cyprus', 'Czechia', 'Denmark', 'Dominican Republic', 'Ecuador', \n", " 'Egypt', 'El Salvador', 'Estonia', 'Finland', 'France', 'Georgia', 'Germany', 'Ghana', 'Greece', \n", " 'Guatemala', 'Honduras', 'Hong Kong', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iraq', 'Ireland', \n", " 'Israel', 'Italy', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kuwait', 'Laos', 'Latvia', \n", " 'Lebanon', 'Libya', 'Liechtenstein', 'Lithuania', 'Luxembourg', 'Malaysia', 'Malta', 'Mexico', \n", " 'Montenegro', 'Morocco', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Nigeria', \n", " 'North Macedonia', 'Norway', 'Oman', 'Pakistan', 'Panama', 'Papua New Guinea', 'Paraguay', 'Peru', \n", " 'Philippines', 'Poland', 'Portugal', 'Puerto Rico', 'Qatar', 'Romania', 'Russia', 'Saudi Arabia', \n", " 'Senegal', 'Serbia', 'Singapore', 'Slovakia', 'Slovenia', 'South Africa', 'South Korea', 'Spain', \n", " 'Sri Lanka', 'Sweden', 'Switzerland', 'Taiwan', 'Tanzania', 'Thailand', 'Tunisia', 'Turkey', 'Uganda', \n", " 'Ukraine', 'United Arab Emirates', 'United Kingdom', 'United States', 'Uruguay', 'Venezuela', 'Vietnam', \n", " 'Yemen', 'Zimbabwe',\n", "]\n", "\n", "ENGLISH_DIALECTS = [\n", " 'Abercraf', 'Aboriginal English in Canada', 'African American English', 'African American Vernacular English', \n", " 'Afro Seminole Creole', 'Angloromani', 'Arablish', 'Atlantic Canadian English', 'Bahamian Creole', \n", " 'Bahamian English', 'Bajan English', 'Baltimore English', 'Bangladeshi English', 'Barrovian', \n", " 'Barrow in Furness', 'Bay Islands English', 'Belizean English', 'Bermudian English', 'Birmingham', \n", " 'Black Country', 'Bolton', 'Boontling', 'Boston', 'Bristolian', 'British English', 'British Indian Empire', \n", " 'Brummie', 'Brunei English', 'Bungi', 'Burmese English', 'Butler English', 'Cajun Vernacular English', \n", " 'California English', 'Cameroonian English', 'Cape Flats English', 'Cardiff', 'Caribbean English', \n", " 'Cayman Islands English', 'Channel Island English', 'Cheshire', 'Chicago', 'Chicano English', \n", " 'Chinese Pidgin English', 'Chinglish', 'Cleveland', 'Cockney', 'Cornwall', 'County Durham', 'County Wexford', \n", " 'Coventry', 'Cumbria', 'Cumbrian', 'Danish English', 'Detroit', 'Dorset', 'Dublin', 'Dutch English', \n", " 'East Anglian', 'East Midlands', 'Eastern New England English', 'Ebonics', 'English in Japan', 'Engrish', \n", " 'Estuary', 'Euro English', 'Falkland Islands English', 'Fingal', 'Fingallian', 'Finnish English', \n", " 'Forth and Bargy dialect', 'Gambian English', 'General American', 'Geordie', 'German English', \n", " 'Ghanaian English', 'Gibraltarian English', 'Glasgow', 'Gower', 'Great Lakes region', \n", " 'Great Northern Coalfield', 'Greater Toronto English', 'Gullah language', 'Guyanese English', \n", " 'Hampshire', 'Hawaiian Pidgin', 'Hiberno English', 'Highland English', 'Hinglish', 'Hoi Toider English', \n", " 'Home Counties', 'Hong Kong English', 'Indian English', 'Inland Northern English', 'Jamaican English', \n", " 'Jamaican Patois', 'Janner', 'Kanglish', 'Kenyan English', 'Konglish', 'Korean English', 'Lancashire', \n", " 'Lancastrian', 'Leinster', 'Liberian English', 'Lincolnshire', 'London', 'Lower Peninsula of Michigan', \n", " 'Lunenburg English', 'Mackem', 'Maine English', 'Malawian English', 'Malaysian English', 'Maltese English', \n", " 'Manchester', 'Mancunian', 'Manglish', 'Manx English', 'Merico language', 'Merseyside', 'Metis', \n", " 'Metropolitan New York English', 'Miami English', 'Mid Atlantic', 'Transatlantic English', \n", " 'Middle Eastern English', 'Middle English', 'Midland American English', 'Milwaukee', 'Multicultural London', \n", " 'Namlish', 'Nepali English', 'New England English', 'New Orleans English', 'New York Latino English', \n", " 'Newfoundland English', 'Nigerian English', 'Norfolk', 'North Central (Upper Midwestern) English', 'Northern', \n", " 'Northern American English', 'Northumberland', 'Northumbrian', 'Norwegian English', \n", " 'Older Southern American English', 'Ottawa Valley English', 'Pacific Northwest English', 'Pakistani English', \n", " 'Pennsylvania Dutch English', 'Philadelphia English', 'Philippine English', 'Pitmatic', 'Port Talbot', \n", " 'Potteries', 'Quebec English', 'Received Pronunciation',\n", " 'Rhode Island English', 'Saban English', 'Saint Helena', 'Scottish English', 'Scouse', 'Sierra Leonean English', \n", " 'Singapore English', 'Singlish', 'Smoggie', 'South African English', 'South Atlantic English', \n", " 'South West Ireland', 'Southern', 'Southern American English', 'Southern Appalachian English', \n", " 'Sri Lankan English', 'Staffordshire', 'Standard Canadian English', 'Suffolk', 'Sunderland', \n", " 'Sussex', 'Swedish English', 'Tanglish', 'Teesside', 'Tenglish', 'Texan English', 'Trinidadian English', \n", " 'Tristan da Cunha', 'Tyneside', 'Ulster', 'Ulster Scots dialects', 'Vincentian Creole', 'Welsh English', \n", " 'West Country', 'West Midlands', 'Western American English', 'Western New England English', 'Western New York', \n", " 'Western Pennsylvania (Pittsburgh) English', 'Yeshiva English', 'Yorkshire',\n", "]\n", "ENGLISH_DIALECTS = sorted(set([re.sub(r\"\\sEnglish$\", \"\", s) for s in ENGLISH_DIALECTS]))" ] }, { "cell_type": "code", "execution_count": 4, "id": "1507a546", "metadata": {}, "outputs": [], "source": [ "with open(\"/data/suno/data/FrequencyWords/word_counts.json\") as f:\n", " word_counts = json.load(f)\n", "\n", "GLOBAL_WORD_COUNTS = {}\n", "for k, v in word_counts.items():\n", " # remove most common words\n", " min_word_count = 50 if k == \"en\" else 3\n", " ignore_top_n = 50 if k == \"en\" else 20\n", " GLOBAL_WORD_COUNTS[k] = [w for w, c in v[ignore_top_n:] if c >= min_word_count and len(w.encode(\"utf8\")) >= 4]\n", "# print(k, \"-\", len(GLOBAL_WORD_COUNTS[k]), \"words\")\n", "\n", "# with open(\"wiki_vocab.json\") as f:\n", "# wiki_vocab_counts = json.load(f)\n", "# wiki_vocab = [k for k, v in wiki_vocab_counts.items() if len(k) >= 4 and v >= 50]\n", "# random.seed(6006)\n", "# random.shuffle(wiki_vocab)\n", "# print(len(wiki_vocab), \"terms\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "063e3586", "metadata": {}, "outputs": [], "source": [ "def get_random_search_sting():\n", " # choose language\n", " if random.random() < 1/2:\n", " lang_code = \"en\"\n", " else:\n", " lang_code = random.choice(list(GLOBAL_WORD_COUNTS.keys()))\n", " # choose word\n", " search_str = random.choice(GLOBAL_WORD_COUNTS[lang_code])\n", " # add dialect / country\n", " if lang_code == \"en\":\n", " if random.random() < 1/3:\n", " search_str += \" \" + random.choice(ENGLISH_DIALECTS)\n", " if random.random() < 1/3:\n", " search_str += \" \" + random.choice(COUNTRY_NAMES)\n", " search_str = search_str.lower()\n", " return lang_code, search_str" ] }, { "cell_type": "code", "execution_count": 6, "id": "c3a50966", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import json\n", "import re\n", "import time\n", "import random\n", "\n", "def _parse_youtube_search_results(html_str):\n", " start_idx = html_str.index(\"ytInitialData\") + len(\"ytInitialData\") + 3\n", " end_idx = html_str.index(\"};\", start_idx) + 1\n", " json_str = html_str[start_idx:end_idx]\n", " page_info = json.loads(json_str)\n", " primary_content = page_info.get(\"contents\", {}).get(\"twoColumnSearchResultsRenderer\", {}).get(\"primaryContents\")\n", " if primary_content is not None:\n", " content_blocks = primary_content[\"sectionListRenderer\"][\"contents\"]\n", " else:\n", " continuation_content = page_info[\"onResponseReceivedCommands\"][0][\"appendContinuationItemsAction\"]\n", " content_blocks = continuation_content[\"continuationItems\"]\n", " results = []\n", " for contents in content_blocks[:-1]:\n", " for video in contents[\"itemSectionRenderer\"][\"contents\"]:\n", " res = {}\n", " if \"videoRenderer\" in video.keys():\n", " video_data = video.get(\"videoRenderer\", {})\n", " res[\"id\"] = video_data.get(\"videoId\", None)\n", " res[\"title\"] = video_data.get(\"title\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"long_desc\"] = video_data.get(\"descriptionSnippet\", {}).get(\"runs\", [{}])[0].get(\"text\", None)\n", " res[\"channel\"] = video_data.get(\"longBylineText\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"duration\"] = video_data.get(\"lengthText\", {}).get(\"simpleText\", None)\n", " res[\"views\"] = video_data.get(\"viewCountText\", {}).get(\"simpleText\", None)\n", " res[\"publish_time\"] = video_data.get(\"publishedTimeText\", {}).get(\"simpleText\", None)\n", " res[\"url_suffix\"] = video_data.get(\n", " \"navigationEndpoint\", {}\n", " ).get(\"commandMetadata\", {}).get(\"webCommandMetadata\", {}).get(\"url\", None)\n", " results.append(res)\n", " if \"continuationItemRenderer\" not in content_blocks[-1]:\n", " continuation_token = None\n", " else:\n", " continuation_endpoint = content_blocks[-1][\"continuationItemRenderer\"][\"continuationEndpoint\"]\n", " continuation_token = continuation_endpoint[\"continuationCommand\"][\"token\"]\n", " return continuation_token, results\n", "\n", "def _get_youtube_search_results(query_term, max_n_pages=None, proxy_url=None):\n", " # NOTE: this returns max ~500 results\n", " query_term = urllib.parse.urlencode({\"a\": query_term})[2:]\n", " if proxy_url is not None:\n", " proxies = {\n", " \"http\": proxy_url,\n", " \"https\": proxy_url,\n", " }\n", " else:\n", " proxies = None\n", " base_url = \"https://www.youtube.com/results?\"\n", " all_results = []\n", " seen_ids = set()\n", " n_pages = 1\n", " while True:\n", " if n_pages == 1:\n", " harvest_url = f\"{base_url}search_query={query_term}&hl=en&sp=EgIoAQ%253D%253D\"\n", " else:\n", " harvest_url = f\"{base_url}continuation={continuation_token}&ctoken={continuation_token}&hl=en\"\n", " out = requests.get(harvest_url, proxies=proxies)\n", " assert(out.ok)\n", " continuation_token, results = _parse_youtube_search_results(out.text)\n", " for m in results:\n", " video_id = m[\"id\"]\n", " if video_id in seen_ids:\n", " continue\n", " seen_ids.add(video_id)\n", " all_results.append(m)\n", " if continuation_token is None:\n", " break\n", " if max_n_pages is not None and n_pages >= max_n_pages:\n", " break\n", " n_pages += 1\n", " _courtesy_sleep(avg_sleep_dur_s=0.3)\n", " return all_results\n", "\n", "def get_youtube_search_results(search_queue_item):\n", " lang_code, search_str, proxy_url = search_queue_item\n", " try:\n", " out = _get_youtube_search_results(search_str, max_n_pages=10, proxy_url=proxy_url)\n", " proxy_country_code = re.search(r\"\\-country\\-(.+?)\\:\", proxy_url).group(1)\n", " domain_str = f\"{lang_code}__{proxy_country_code}__{search_str}\"\n", " assert(len(out) > 0)\n", " return domain_str, out\n", " except Exception as e:\n", " out_meta = None\n", " return None" ] }, { "cell_type": "code", "execution_count": 7, "id": "04009a66", "metadata": {}, "outputs": [], "source": [ "search_queue_items = []\n", "for _ in range(100_000):\n", " lang_code, search_str = get_random_search_sting()\n", " proxy_url = get_global_proxy_url()\n", " search_queue_items.append((lang_code, search_str, proxy_url))" ] }, { "cell_type": "code", "execution_count": 15, "id": "f90ee2ef", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [33:53:17<00:00, 609.99s/it]\n" ] } ], "source": [ "# ~10 mins for 500 terms (10 cores) --> 2 days for all 100k search terms\n", "_ = mp_scrape(\n", " get_youtube_search_results, \n", " search_queue_items, \n", " chunksize=500,\n", " n_cores=10,\n", " n_retries=1,\n", "# continue_partial=True,\n", " result_filepath=os.path.join(DATA_DIR, \"videos_by_domain_global.jsonl\"), \n", " log_filepath=os.path.join(LOGS_DIR, \"video_urls_harvest.log\"),\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "16c33484", "metadata": {}, "outputs": [], "source": [ "# TODO: RUNNING THIS HERE" ] }, { "cell_type": "code", "execution_count": 18, "id": "18ab984c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100000it [00:57, 1726.41it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "12620995 videos found\n", "6831596 unique\n", "1668065.9 hours of data\n", "1339629.9 hours of data with clipping\n" ] } ], "source": [ "youtube_ids = set()\n", "tot_n = 0\n", "durations_s = []\n", "failed_ids = []\n", "with open(os.path.join(DATA_DIR, \"videos_by_domain_global.jsonl\")) as f:\n", " for l in tqdm.tqdm(f):\n", " if len(l.strip()) == 0:\n", " continue\n", " m = json.loads(l)\n", " if m is None:\n", " continue\n", " domain_str, search_results = m\n", " for search_result in search_results:\n", " if search_result[\"id\"] not in youtube_ids:\n", " try:\n", " ds = _parse_duration(search_result[\"duration\"])\n", " except:\n", " failed_ids.append(search_result[\"id\"])\n", " continue\n", " durations_s.append(ds)\n", " youtube_ids.add(search_result[\"id\"])\n", " tot_n += 1\n", "youtube_ids = list(youtube_ids)\n", "print(tot_n, \"videos found\")\n", "print(len(youtube_ids), \"unique\")\n", "print(len(failed_ids), \"failed\")\n", "print(round(np.sum(durations_s) / 60 / 60, 1), \"hours of data\")\n", "print(round(np.clip(np.array(durations_s), a_max=3600, a_min=0).sum() / 60 / 60, 1), \"hours of data with clipping\")" ] }, { "cell_type": "code", "execution_count": null, "id": "afb97fcb", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7f3c5458", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a8e27363", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3af3d019", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }