{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "6304ee0e", "metadata": {}, "outputs": [], "source": [ "import re\n", "import json\n", "import random\n", "import tqdm\n", "\n", "from suno_utils.harvest.utils import mp_scrape\n", "from suno_utils.harvest.youtube.collect import get_search_results, time_limit, parse_duration, TimeoutException\n", "from suno_utils.harvest.youtube.constants.base import SUPPORTED_LANGS\n", "from suno_utils.harvest.youtube.constants.harvest import HL_LANGS\n", "\n", "PROXY_URL = (\n", " \"http://brd-customer-hl_98887cab-zone-us_proxy-route_err-block-country-us:\" +\n", " \"3if5he8elfe7@zproxy.lum-superproxy.io:22225\"\n", ")\n", "\n", "DATA_DIR = \"/data2/suno/data/harvest/youtube_ml\"" ] }, { "cell_type": "code", "execution_count": 2, "id": "cb3c419c", "metadata": {}, "outputs": [], "source": [ "import youtube_dl\n", "from contextlib import redirect_stderr, redirect_stdout\n", "import os\n", "\n", "DEFAULT_GET_SUBTITLES_TIMEOUT_S = 10.0\n", "YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n", "\n", "def get_subtitles(youtube_id, timeout_s=DEFAULT_GET_SUBTITLES_TIMEOUT_S, proxy_url=None):\n", " ydl_options = {\n", " \"listallsubtitles\": True,\n", " \"socket_timeout\": 5.0,\n", " }\n", " if proxy_url is not None:\n", " ydl_options[\"proxy\"] = proxy_url\n", " url = YOUTUBE_BASE_URL + youtube_id\n", " with redirect_stderr(open(os.devnull, \"w\")):\n", " with redirect_stdout(open(os.devnull, \"w\")):\n", " with youtube_dl.YoutubeDL(ydl_options) as ydl:\n", " if timeout_s is not None:\n", " with time_limit(timeout_s):\n", " info = ydl.extract_info(url, download=False)\n", " else:\n", " info = ydl.extract_info(url, download=False)\n", " return sorted(list(set(info[\"subtitles\"].keys())))" ] }, { "cell_type": "code", "execution_count": 3, "id": "b3c02b30", "metadata": {}, "outputs": [], "source": [ "# get seed terms\n", "with open(os.path.join(DATA_DIR, \"query_terms\", \"unigrams.json\")) as f:\n", " seed_terms = json.load(f)" ] }, { "cell_type": "code", "execution_count": 4, "id": "0dbacd9b", "metadata": {}, "outputs": [], "source": [ "# filter based on summary attributes\n", "DEFAULT_MIN_DURATION_S = 30\n", "DEFAULT_MAX_DURATION_S = 2 * 60 * 60\n", "DEFAULT_MIN_VIEWS = 50\n", "def filter_by_meta(\n", " search_results, \n", " min_duration_s=DEFAULT_MIN_DURATION_S, \n", " max_duration_s=DEFAULT_MAX_DURATION_S, \n", " min_views=DEFAULT_MIN_VIEWS,\n", "):\n", " filtered_results = []\n", " for m in search_results:\n", " try:\n", " duration_s = parse_duration(m[\"duration\"])\n", " if duration_s < min_duration_s or duration_s > max_duration_s:\n", " continue\n", " views = int(re.sub(r\"[^0-9]\", \"\", m[\"views\"]))\n", " if views < min_views:\n", " continue\n", " filtered_results.append(m)\n", " except:\n", " continue\n", " return filtered_results\n", "\n", "# filter based on CC files\n", "DEFAULT_MAX_N_SUBTITLES = 5\n", "def filter_by_subtitles(search_results, lang_code, max_n_subtitles=DEFAULT_MAX_N_SUBTITLES, proxy_url=None):\n", " filtered_results = []\n", " for m in search_results:\n", " try:\n", " subtitle_langs = get_subtitles(m[\"id\"], proxy_url=proxy_url)\n", " except:\n", " continue\n", " if len(subtitle_langs) > max_n_subtitles:\n", " continue\n", " if not any([subtitle_lang.split(\"-\")[0] == lang_code for subtitle_lang in subtitle_langs]):\n", " continue\n", " filtered_results.append(m)\n", " return filtered_results" ] }, { "cell_type": "code", "execution_count": null, "id": "3fd38182", "metadata": {}, "outputs": [], "source": [ "# af confirmed: FPSMUdDJtG8, YVGK-eeJ8qo, pOnUYZecPCg, jbqdhWsdIFE, sqtHMqpRPSQ\n", "# https://genius.com/Andriette-druppel-in-die-see-lyrics" ] }, { "cell_type": "code", "execution_count": 6, "id": "d1dee57e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0 1 2 (0)\n", "0 1 5 (0)\n", "0 1 20 (6)\n", "0 2 2 (0)\n", "0 2 5 (0)\n", "0 2 20 (0)\n", "0 3 2 (0)\n", "0 3 5 (0)\n", "0 3 20 (0)\n", "1 1 2 (1)\n", "1 1 5 (4)\n", "1 1 20 (13)\n", "1 2 2 (2)\n", "1 2 5 (0)\n", "1 2 20 (0)\n", "1 3 2 (0)\n", "1 3 5 (0)\n", "1 3 20 (0)\n", "2 1 2 (0)\n", "2 1 5 (0)\n", "2 1 20 (0)\n", "2 2 2 (0)\n", "2 2 5 (1)\n", "2 2 20 (0)\n", "2 3 2 (0)\n", "2 3 5 (0)\n", "2 3 20 (0)\n", "3 1 2 (0)\n", "3 1 5 (0)\n", "3 1 20 (0)\n", "3 2 2 (0)\n", "3 2 5 (0)\n", "3 2 20 (0)\n", "3 3 2 (0)\n", "3 3 5 (0)\n", "3 3 20 (0)\n", "4 1 2 (0)\n", "4 1 5 (1)\n", "4 1 20 (0)\n", "4 2 2 (2)\n", "4 2 5 (0)\n", "4 2 20 (0)\n", "4 3 2 (0)\n", "4 3 5 (0)\n", "4 3 20 (0)\n", "5 1 2 (0)\n", "5 1 5 (0)\n", "5 1 20 (1)\n", "5 2 2 (0)\n", "5 2 5 (0)\n", "5 2 20 (0)\n", "5 3 2 (0)\n", "5 3 5 (0)\n", "5 3 20 (0)\n", "6 1 2 (0)\n", "6 1 5 (0)\n", "6 1 20 (0)\n", "6 2 2 (0)\n", "6 2 5 (0)\n", "6 2 20 (0)\n", "6 3 2 (0)\n", "6 3 5 (0)\n", "6 3 20 (0)\n", "7 1 2 (0)\n", "7 1 5 (0)\n", "7 1 20 (0)\n", "7 2 2 (0)\n", "7 2 5 (0)\n", "7 2 20 (0)\n", "7 3 2 (0)\n", "7 3 5 (0)\n", "7 3 20 (0)\n", "8 1 2 (0)\n", "8 1 5 (0)\n", "8 1 20 (0)\n", "8 2 2 (0)\n", "8 2 5 (0)\n", "8 2 20 (0)\n", "8 3 2 (0)\n", "8 3 5 (0)\n", "8 3 20 (0)\n", "9 1 2 (0)\n", "9 1 5 (0)\n", "9 1 20 (0)\n", "9 2 2 (0)\n", "9 2 5 (1)\n", "9 2 20 (0)\n", "9 3 2 (0)\n", "9 3 5 (0)\n", "9 3 20 (0)\n" ] } ], "source": [ "# TODO: testing which type of stuff works best\n", "lang_code = \"af\"\n", "hl_lang = \"af\"\n", "data = []\n", "for n in range(10):\n", " for n_gram in [1, 2, 3]:\n", " for n_pages in [2, 5, 20]:\n", " query_terms = []\n", " for _ in range(n_gram):\n", " query_terms.append(random.choice(seed_terms[lang_code]))\n", " query_term = \" \".join(query_terms)\n", " search_results = get_search_results(query_term, hl=hl_lang, max_n_pages=n_pages, proxy_url=PROXY_URL)\n", " promising_results = filter_by_meta(search_results)\n", " reliable_results = filter_by_subtitles(promising_results, lang_code, proxy_url=PROXY_URL)\n", " data.append((n_gram, n_pages, search_results, [m[\"id\"] for m in reliable_results]))\n", " print(n, n_gram, n_pages, f\"({len(reliable_results)})\")" ] }, { "cell_type": "code", "execution_count": null, "id": "ba2f5080", "metadata": {}, "outputs": [], "source": [ "# TODO: why all later ones failed? retry on certain exceptions?\n", "\n", "# TODO: get more pages for 'good' words?\n", "# TODO: get channel of successful ones and keep exploring\n", "# TODO: get ones with > 5 subtitles as LM corpus?" ] }, { "cell_type": "code", "execution_count": null, "id": "bd2f81eb", "metadata": {}, "outputs": [], "source": [ "# TODO: get song count for all languages we support from here:\n", "# /data/suno/data/harvest/genius/song_details_with_youtube.jsonl\n", "# \"lang\": \"en\"" ] }, { "cell_type": "code", "execution_count": 14, "id": "e6f5085f", "metadata": {}, "outputs": [], "source": [ "l = []\n", "with open(\"/data/suno/data/harvest/genius/song_details_with_youtube.jsonl\") as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " l.append(m[\"lang\"])" ] }, { "cell_type": "code", "execution_count": 32, "id": "29edc333", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['Zsi5HqBQOU8', '5Q__nuMQBlI', '8B1XSCELoAQ', 'PJIQjYLY2wo', 'c7Gm6BMpgeE', 'XuFYjLaSP7I', 'VOw3gt_ydJE', 'nogD2JbfXbY', 'wjoiupJZCr0', 'hqJl_OPNgyA', 'wu1_XCKh58Q', 'oG6IRIvnz3k', 'AL_QjPwI9kQ', 'h1oM2ff0lQY', 'WUZwXh7XS4I', 'Ad0JtEUp1XI', 'dKvBT5mXBvI', '83Abu_fs9EE', 'G_ScIfKpq4I', 'LVClMgnPV4g', 'lzWPFTSdft8', 'BcrkohG_lkQ', 'VDgrBMMtDDk', 'OUuZjmq5iMQ', 'W9gOyqOVHDw', 'N8MdmvQjXfI', 'Jrdxm_dbKFo', 'yG12ylKjOTg', 'WWBX1m8NY4M', 'vO8R6xHRoZc', 'SM9wwziHSa8', '3Dr40svKCs0']\n" ] } ], "source": [ "print(list(set(funcy.flatten([e[-1] for e in data]))))" ] }, { "cell_type": "code", "execution_count": null, "id": "06aecfa4", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3d00e77d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "030061e6", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 25, "id": "3fd1cc47", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "54 - 52 - 1 -- gestig -- ['AhfsMT7mpNw']\n", "59 - 54 - 6 -- dikwels -- ['l1NxVBPe3W8', '9Q7RQxOLJcA', '7Nhr8WNh1LM', 'ZyiG9UsX4vI', 'O2e8LsNt_2s', 'qGMz5QyoSRk']\n", "49 - 49 - 6 -- plaas -- ['BNXOkRRRp4Q', 'pOnUYZecPCg', 'fULIf13fFzU', '4EZkpjwqC0Y', 'VTrgCImGJvs', 'IMAAmQiQePw']\n", "58 - 44 - 1 -- Amerikaanse -- ['usglkd5jlhk']\n", "55 - 44 - 2 -- steeds -- ['Zpw1PZiXG7Y', 'UgkBU4Eb5YI']\n", "58 - 52 - 0 -- sluit -- []\n", "55 - 54 - 1 -- waarin -- ['XUBU3CZ1piU']\n", "59 - 53 - 8 -- Gereformeerde -- ['jbqdhWsdIFE', 'IebwFgrEn70', 'uXMaXeIqkMs', 'UwogvpJIu-k', '3vH0XNearvQ', 'hgtEhCRGNyQ', 'RVcxKLTn-M0', 'L_abzpJPW1U']\n", "56 - 56 - 0 -- water -- []\n", "54 - 54 - 0 -- Franse -- []\n", "55 - 48 - 0 -- geleë -- []\n", "55 - 46 - 6 -- verwys -- ['qv5pElBz10Q', 'L6xVe9OcSHw', 'I1zIhkYKHH4', 'SQ_q2X2ty6E', 'sqtHMqpRPSQ', 'UarXMuZDDx0']\n", "59 - 55 - 0 -- einde -- []\n", "53 - 51 - 4 -- Kaapstad -- ['UbKdj13Z81A', 'vUzRO5yL6Cs', '9X8UBIbQN7I', 'cy9VmtaOM-c']\n", "54 - 53 - 0 -- Universiteit -- []\n", "51 - 48 - 0 -- wanneer -- []\n", "51 - 51 - 17 -- Afrikaans -- ['uI49IqDCgg8', 'vCKmpBfSOeY', 'SVidugsl-FY', 'X73A0x4WDhA', 'T_8snLihxWs', 'GUCiOlJN2sU', 'IUyt2O7eURg', 'FPSMUdDJtG8', '1rSxMOFeYr4', 'KkTFYSfodKk', 'EYgjMMtG-ek', 'vWRfMhU_9c0', 'GrHs-OxvgfA', 'uxOe4UJAsCQ', 'BHhgLzuudzE', '-Pgdo8Fu7Ag', 'H6BSqQskpBk']\n", "56 - 52 - 3 -- besluit -- ['sDinYG1yw7o', 'AH1K0RXPm3c', 'h7e9Fhwwu08']\n", "53 - 37 - 0 -- aantal -- []\n", "57 - 39 - 0 -- venster -- []\n", "47 - 38 - 5 -- asook -- ['UarXMuZDDx0', 'ZKIFbgjc_QI', 'p4kExgmN0jY', 'FNdh9BAvEgA', 'zzsFMQS7eT4']\n", "51 - 49 - 5 -- daarvan -- ['Co9X5ktK4cI', 'quI3hKDDXnI', 'Uc1Dr2RsiTE', 'w50UL9haZcg', 'FjKNBfTX5ZY']\n", "51 - 49 - 0 -- miljoen -- []\n", "52 - 51 - 1 -- laaste -- ['h2ytMc-4R_o']\n", "52 - 46 - 0 -- nadat -- []\n", "59 - 55 - 4 -- verskillende -- ['R9R0S16iwDE', 'WfZ02nL59mo', 'l1NxVBPe3W8', 'w-xt2kqRMHQ']\n", "44 - 39 - 0 -- Pretoria -- []\n", "56 - 56 - 4 -- gewoonlik -- ['w50UL9haZcg', 'I1zIhkYKHH4', 'xX4cIkXpB_I', 'Pv72yDr09hw']\n", "52 - 48 - 0 -- sedert -- []\n", "50 - 49 - 0 -- sonder -- []\n", "57 - 54 - 0 -- koning -- []\n", "51 - 48 - 0 -- enige -- []\n", "56 - 56 - 0 -- selfs -- []\n", "56 - 50 - 0 -- leraar -- []\n", "56 - 47 - 7 -- ontwikkel -- ['j2vD8Wy56n4', 'C1jYPYy3JUs', '5_UD36xPi8Y', 'nLxLqFB-DwM', 'nS9w8Oiyv0E', '_mJ1i31eusc', 'LUUwM0SvRas']\n", "51 - 50 - 7 -- moontlik -- ['Jis9tufkqT8', 'lBrmenI6GfQ', 'GeowQvP032g', 'Wmpk3BqS8OM', 'XWGSIHBhQ2Y', 'ir2uXTNHlec', 'h1oM2ff0lQY']\n", "56 - 32 - 0 -- Engelse -- []\n", "56 - 55 - 0 -- Frankryk -- []\n", "53 - 48 - 0 -- politieke -- []\n", "57 - 54 - 0 -- ontstaan -- []\n", "55 - 49 - 1 -- Tweede -- ['_cldfWLRuDs']\n", "51 - 38 - 4 -- skrywer -- ['4CNc8532SjU', '5FiWNhMOQpU', 'sqtHMqpRPSQ', '-1qgH-QH0vo']\n", "53 - 52 - 0 -- daardie -- []\n", "54 - 54 - 4 -- Nasionale -- ['gSaUWQrKclQ', 'Kp1bW8-x5ZE', 'wmSqPg4yhMU', '3Vg6H8B4pHc']\n", "55 - 53 - 0 -- speel -- []\n", "52 - 49 - 1 -- belangrike -- ['g_RqDu4wx8w']\n", "56 - 49 - 0 -- ontvang -- []\n", "54 - 53 - 0 -- daarna -- []\n", "52 - 52 - 1 -- geneem -- ['SVidugsl-FY']\n", "49 - 40 - 1 -- maande -- ['TNqPwcHNstc']\n" ] } ], "source": [ "lang_code = \"af\"\n", "hl_lang = \"af\"\n", "data_1 = []\n", "for query_term in seed_terms[lang_code][5:55]:\n", " search_results = get_search_results(query_term, hl=hl_lang, max_n_pages=3, proxy_url=PROXY_URL)\n", " promising_results = filter_by_meta(search_results)\n", " reliable_results = filter_by_subtitles(promising_results, lang_code, proxy_url=PROXY_URL)\n", " print(\n", " len(search_results), \"-\", \n", " len(promising_results), \"-\", \n", " len(reliable_results), \"--\", \n", " query_term, \"--\",\n", " [m[\"id\"] for m in reliable_results]\n", " )\n", " data_1.append((query_term, search_results, reliable_results))" ] }, { "cell_type": "code", "execution_count": null, "id": "ccd77f28", "metadata": {}, "outputs": [], "source": [ "# TODO: would classifying the title language help?\n", "# TODO: would unique mapping help?" ] }, { "cell_type": "code", "execution_count": null, "id": "21ba0b63", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a5713484", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a24c5d5b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\r", " 0%| | 0/10 [00:00= MAX_IDS_PER_LANG:\n", " continue\n", " if len(seed_terms[lang_code]) <= n_iter:\n", " continue\n", " # TODO: do we randomize here or take most common term?\n", " query_term = seed_terms[lang_code][n_iter]\n", " # resolve into HL language\n", " available_hl_langs = [k for k in HL_LANGS.keys() if k.split(\"-\")[0] == lang_code]\n", " if len(available_hl_langs) == 0:\n", " hl_lang = \"en\"\n", " else:\n", " hl_lang = random.choice(available_hl_langs)\n", " try:\n", " # TODO: test how many pages we want? different sorting for low ressource languages?\n", " search_results = get_search_results(query_term, hl=hl_lang, max_n_pages=MAX_N_PAGES, proxy_url=PROXY_URL)\n", " except TimeoutException:\n", " continue\n", " promising_results = filter_by_meta(search_results)\n", " try:\n", " reliable_results = filter_by_subtitles(promising_results, lang_code, proxy_url=PROXY_URL)\n", " except TimeoutException:\n", " continue\n", " # TODO: download vtt files to a directory?\n", " reliable_ids = [m[\"id\"] for m in reliable_results]\n", " counts_per_lang[lang_code] += len(reliable_ids)\n", " data[lang_code].append((hl_lang, search_results, reliable_ids))\n", " if all(counts_per_lang[lang_code] >= MAX_IDS_PER_LANG for lang_code in SUPPORTED_LANGS):\n", " break\n", " \n", " print(counts_per_lang)\n", "# TODO: add mp and retries/logging/etc" ] }, { "cell_type": "code", "execution_count": null, "id": "c203a229", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "ab6c7e09", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 53, "id": "01b7628c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n" ] } ], "source": [ "import fasttext\n", "fasttext_lang_model = fasttext.load_model(\"lid.176.bin\")" ] }, { "cell_type": "code", "execution_count": 69, "id": "3cd324ae", "metadata": {}, "outputs": [], "source": [ "from suno_utils.harvest.youtube.constants.text_lang import FASTTEXT_LANGS, BASE_TO_FASTTEXT_REMAP\n", "\n", "FASTTEXT_TO_BASE_REMAP = {k: v for k, v in BASE_TO_FASTTEXT_REMAP.items()}\n", "\n", "def _lang_classify_text_clean(text):\n", " text = text.replace(\"’\", \"'\")\n", " text = text.lower()\n", " return text\n", "\n", "\n", "def get_text_lang_p(fasttext_lang_model, check_lang_code, text, top_k=-1):\n", " \"\"\"get probability of input language for text\"\"\"\n", " text = _lang_classify_text_clean(text)\n", " out = fasttext_lang_model.predict(text, k=top_k)\n", " p_dict = {}\n", " for lang_str, p in zip(*out):\n", " lang = lang_str.split(\"__\")[-1]\n", " lang = FASTTEXT_TO_BASE_REMAP.get(lang, lang)\n", " p_dict[lang] = p\n", " return p_dict.get(check_lang_code, 0.0)" ] }, { "cell_type": "code", "execution_count": 72, "id": "84197f03", "metadata": {}, "outputs": [], "source": [ "data = []\n", "for search_result in search_results:\n", " p_lang = get_text_lang_p(fasttext_lang_model, \"af\", search_result[\"title\"])\n", " data.append((p_lang, search_result[\"id\"]))" ] }, { "cell_type": "code", "execution_count": 74, "id": "916916cf", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(0.7478622794151306, 'AEOyHUXhGog'),\n", " (0.6945279836654663, 'hmEd-BB9nZc'),\n", " (0.5681423544883728, 'FW80LTLCZRE'),\n", " (0.2509119212627411, 'Kx8iOJaQmPQ'),\n", " (0.09074866026639938, 'RSIXmiuW074'),\n", " (0.005381728056818247, 'q3M_rdef7sw'),\n", " (0.003784764092415571, 'AWlkGbFnS80'),\n", " (0.003765847533941269, 'DPJGJ5XOLFQ'),\n", " (0.0036642858758568764, '5DjPWz0lS3g'),\n", " (0.0036099939607083797, '9sGmUv7Abuo')]" ] }, "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sorted(data, reverse=True, key=lambda x: x[0])[:10]" ] }, { "cell_type": "code", "execution_count": 79, "id": "919708b9", "metadata": {}, "outputs": [], "source": [ "out = filter_by_subtitles(search_results, proxy_url=PROXY_URL)" ] }, { "cell_type": "code", "execution_count": 80, "id": "57b4692c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2" ] }, "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(out)" ] }, { "cell_type": "code", "execution_count": 78, "id": "abc8d7e4", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(0.00353, 'UUhczekj93Y', 0)\n", "(0.00338, 'MVSwN2U18Xw', 0)\n", "(0.00315, 'tQSvxT7uYQ4', 0)\n", "(0.00315, 'NOXZ5_ISstk', 0)\n", "(0.00313, 'ywWjcRaE8ZQ', 0)\n", "(0.00307, '-qIxhNe5_S0', 0)\n", "(0.00282, 'degA6NgpBXo', 0)\n", "(0.00282, 'Sv7dZ5VZpA0', 0)\n", "(0.00274, 'MDukRsiFM_s', 0)\n", "(0.00261, 'doxwwYKvlfw', 0)\n", "(0.00253, 'Uqybp9uKsug', 0)\n", "(0.00248, 'RUAsmwYC2mc', 0)\n", "(0.00234, 'ecWNflDzFfA', 0)\n", "(0.00233, 'nMPWpPd6JOs', 0)\n", "(0.00233, '3jQSXQhsNQk', 0)\n", "(0.00232, 'piAwuADrC3Y', 0)\n", "(0.00225, '9OwNGXsrGw0', 0)\n", "(0.00217, '9WsyLL6KVBY', 0)\n", "(0.00216, 'hyDlPjuUEKw', 0)\n", "(0.00213, 'jZJBvmt59jk', 0)\n", "(0.00213, 'WCpKDSmiodM', 0)\n", "(0.00213, '52hwzUaeg88', 0)\n", "(0.00201, 'YiiHNiSfmKI', 0)\n", "(0.00201, 'FT9DMQEv8yU', 0)\n", "(0.0019, 'nNHMNx0I33M', 0)\n", "(0.0019, 't2BKWFyJOXw', 0)\n", "(0.00188, 'FO5OQL8mN08', 0)\n", "(0.00188, '_7tEb94b18o', 0)\n", "(0.00178, 'GG44px1LfHc', 0)\n", "(0.00172, 'D9Ihs241zeg', 0)\n", "(0.00171, '8Ba7SgCN2XY', 0)\n", "(0.00167, 'sHRHkX9J0eA', 0)\n", "(0.00163, 'YXDvIMWBFcM', 0)\n", "(0.00156, 'EpWfDdY4-vk', 0)\n", "(0.00155, '8ID9UYzGess', 0)\n", "(0.00135, 'qYhVDf6cB4o', 0)\n", "(0.00132, 'Yj1TwQywaIE', 0)\n", "(0.00126, 'SfVSM_Ow9rU', 0)\n", "(0.00117, 'lLOD8QP4nXo', 0)\n", "(0.00112, 'gJWM3pBbh0Q', 0)\n", "(0.0011, 'KWxwYbaAWxs', 0)\n", "(0.00103, 'lLUhOxJxpGw', 0)\n", "(0.00099, 'MuQyMK6n_k8', 0)\n", "(0.00086, 'wp5FVHMXEbA', 0)\n", "(0.00083, 'HkQD6aw7oDc', 0)\n", "(0.00075, 'r9LCwI5iErE', 0)\n", "(0.00072, '8D9gGxS1_qg', 0)\n", "(0.0007, 'Hv5JfMDLgCU', 0)\n", "(0.00064, 'j7kYVUU5_DE', 0)\n", "(0.00064, '2aP9Azs_Kac', 0)\n", "(0.00059, 'arRDKrVGw7k', 0)\n", "(0.00059, 'XwhZ4xX7Qmc', 0)\n", "(0.00059, 'hilxtgjay_U', 0)\n", "(0.00058, 'EXSijgdztCg', 0)\n", "(0.00058, 'KGZFmW3uPJE', 0)\n", "(0.00055, 'gcx-RThIKic', 0)\n", "(0.00054, 'XdCxfJki4t4', 0)\n", "(0.00054, '8Qj8aUATBOc', 0)\n", "(0.00052, 'FcIVPVLt2L4', 0)\n", "(0.00052, 'wQtftfwYiO4', 0)\n", "(0.00051, 'c3yJhw7R3fI', 0)\n", "(0.00051, 'ksu-zTG9HHg', 0)\n", "(0.00048, 'S-_KiyVCZ_8', 0)\n", "(0.00041, 'tIW8vXytTzM', 0)\n", "(0.00041, 'F5nlc2wwvvI', 0)\n", "(0.0004, '3VKycCiBmcU', 0)\n", "(0.00039, 'BDPnMxec_UM', 0)\n", "(0.00039, 'M_5L0mnf53s', 0)\n", "(0.00039, 'fUhiHTHtYa4', 0)\n", "(0.00039, 'G3AvaCpH_RU', 0)\n", "(0.00037, '2kTPHDHrPUA', 0)\n", "(0.00037, 'VbKjgeryttg', 0)\n", "(0.00037, '4rAX8mFfZV4', 0)\n", "(0.00036, '4LZWAF0V2xI', 0)\n", "(0.00035, 'p-j8tTxVdDU', 0)\n", "(0.00034, '0Oc71RBiNd0', 0)\n", "(0.00033, 't9yHEibkUbE', 0)\n", "(0.00033, 'pzmB0GoEKkA', 0)\n", "(0.00032, 'Egz5lwsZULw', 0)\n", "(0.00032, '9wf9lO3Vos0', 0)\n", "(0.00031, 'XSOVfpapSXA', 0)\n", "(0.00031, '2-sgjxO5rvQ', 0)\n", "(0.00031, 'tTex2yUTXEs', 0)\n", "(0.00029, 'YOJU8txW8GA', 0)\n", "(0.00029, '6Vo2njedRmA', 0)\n", "(0.00027, '3GntDYJx8JA', 0)\n", "(0.00026, 'ZD4BcbtE7W0', 0)\n", "(0.00026, '14hJvcR2M-o', 0)\n", "(0.00025, 'OQRX8Ijg7yY', 0)\n", "(0.00024, 'YNfzYOD6L2Y', 0)\n", "(0.00024, 'BWRz9kfsOrI', 0)\n", "(0.00024, 'gCGZZR48fAs', 0)\n", "(0.00024, 'Cx2TlBjsBOo', 0)\n", "(0.00023, 'V7upyOY4Xrs', 0)\n", "(0.00022, '8u-q9e3SrP8', 0)\n", "(0.00021, 'JZW9UEF-DmU', 0)\n", "(0.00021, 'l4PTEP4hh-E', 0)\n", "(0.00021, 'dRvtOmCL_W0', 0)\n", "(0.0002, 'quI3hKDDXnI', 1)\n", "(0.0002, 'QxcvMX3HtIE', 0)\n", "(0.00019, 'IFm-9hSF05E', 0)\n", "(0.00018, 'JwJPJHl_a-Y', 0)\n", "(0.00016, 'sNNRo7KuFHY', 0)\n", "(0.00016, 'lxH7rakMQTU', 0)\n", "(0.00015, '0rYyoQIIxDI', 0)\n", "(0.00015, 'mvFWoIska9w', 0)\n", "(0.00014, 'asFykTYK1Bs', 0)\n", "(0.00014, 'zvFvFD8Hz4U', 0)\n", "(0.00012, 'SIVim0nF4p8', 0)\n", "(0.00012, 'AMyZuFqXhTE', 0)\n", "(0.00012, 'H5kp8w86syg', 0)\n", "(0.00011, '7kfQseZPCJE', 0)\n", "(0.00011, '9dZzyl7dCuw', 0)\n", "(0.0001, 'i42GUZbKTPU', 0)\n", "(0.0001, '4t7GYI5nw74', 0)\n", "(9e-05, 'jGWmiShZRSQ', 0)\n", "(9e-05, 'ySrqjemVe7g', 0)\n", "(8e-05, 'E-BOLfexNBM', 0)\n", "(8e-05, 'a4aMYy1kq_c', 0)\n", "(8e-05, 'GjcHmboOyrs', 0)\n", "(8e-05, 'atBMLJdSKBo', 0)\n", "(7e-05, 'h0OncH8NaqQ', 0)\n", "(7e-05, 'GnWH786zjgc', 0)\n", "(7e-05, 'gPjOL1zhFjY', 0)\n", "(7e-05, 'xIEYYrAeZyE', 0)\n", "(7e-05, 'Y-GaINhkoVY', 0)\n", "(7e-05, '8M09QzqzhIk', 0)\n", "(7e-05, 'EfgW-SB_DoQ', 0)\n", "(6e-05, '_LRp8fB69Iw', 0)\n", "(6e-05, 'kD11wmFrUJA', 0)\n", "(6e-05, 'lw1U7CvmjoU', 0)\n", "(6e-05, 'xR0Wg87C-js', 0)\n", "(5e-05, 'vVnFWhFuDZ0', 0)\n", "(5e-05, 'joab10an3lg', 0)\n", "(5e-05, 'LbDYLUuwnTk', 0)\n", "(5e-05, 'YR_tckUfcEc', 0)\n", "(4e-05, 'iwp1BtRQVVE', 0)\n", "(4e-05, 'Gy9vc6h7OtQ', 0)\n", "(4e-05, 'kNOgM0ForHU', 0)\n", "(4e-05, 'g1whQJIqDEo', 0)\n", "(4e-05, 'uYiVtZHns-A', 0)\n", "(4e-05, 'KUB-aJXquUA', 0)\n", "(3e-05, 'UuFWYOnHwGM', 0)\n", "(3e-05, 'BDhh-Qeboho', 0)\n", "(3e-05, 'vBaHg2-zXM4', 0)\n", "(3e-05, 'Y4vV61lbL3g', 0)\n", "(3e-05, 'SikbOMVOJYQ', 0)\n", "(2e-05, 'yoDrhOZCKWo', 0)\n", "(2e-05, '9GK-7ZpKnxA', 0)\n", "(2e-05, 'Y78E_hAKnsw', 0)\n", "(1e-05, 'QP2hZLkhq30', 0)\n", "(0.0, 'UZpkKIFs3PQ', 0)\n", "(0.0, 'NdxV4VBTsek', 0)\n", "(0.0, 'C61-8EEtgYQ', 0)\n", "(0.0, 'qla-KaMF-2Q', 0)\n", "(0.0, 'LKPvQ92WMl4', 0)\n", "(0.0, 'e7PQpwUbpk0', 0)\n", "(0.0, '1v908bm40HI', 0)\n", "(0.0, 'J6NVOMCV3W0', 0)\n", "(0.0, 'zdpLbvCcq8Q', 0)\n", "(0.0, '4_p4TQIXts4', 0)\n", "(0.0, 'Gedsiv9KIqo', 0)\n" ] } ], "source": [ "for p, _id in sorted(data, reverse=True, key=lambda x: x[0])[10:]:\n", " check_results = [m for m in search_results if m[\"id\"] == _id]\n", " print((round(p, 5), _id, len(filter_by_subtitles(check_results, proxy_url=PROXY_URL))))" ] }, { "cell_type": "code", "execution_count": null, "id": "55aff976", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3ac9a664", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "900bf6f7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "18130669", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b56dd56a", "metadata": {}, "outputs": [], "source": [ "# -*- coding: utf-8 -*-\n", "\n", "# Sample Python code for youtube.search.list\n", "# See instructions for running these code samples locally:\n", "# https://developers.google.com/explorer-help/code-samples#python\n", "\n", "import os\n", "\n", "import google_auth_oauthlib.flow\n", "import googleapiclient.discovery\n", "import googleapiclient.errors\n", "\n", "scopes = [\"https://www.googleapis.com/auth/youtube.force-ssl\"]\n", "\n", "def main():\n", " # Disable OAuthlib's HTTPS verification when running locally.\n", " # *DO NOT* leave this option enabled in production.\n", " os.environ[\"OAUTHLIB_INSECURE_TRANSPORT\"] = \"1\"\n", "\n", " api_service_name = \"youtube\"\n", " api_version = \"v3\"\n", " client_secrets_file = \"YOUR_CLIENT_SECRET_FILE.json\"\n", "\n", " # Get credentials and create an API client\n", " flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(\n", " client_secrets_file, scopes)\n", " credentials = flow.run_console()\n", " youtube = googleapiclient.discovery.build(\n", " api_service_name, api_version, credentials=credentials)\n", "\n", " request = youtube.search().list(\n", " part=\"snippet\",\n", " q=\"skryf\",\n", " relevanceLanguage=\"af\"\n", " )\n", " response = request.execute()\n", "\n", " print(response)\n", "\n", "if __name__ == \"__main__\":\n", " main()" ] }, { "cell_type": "code", "execution_count": null, "id": "cab5c39b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "466488c8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "07200477", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e327c80b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7a741334", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5b833b96", "metadata": {}, "outputs": [], "source": [ "# YDL_OPTS = {\n", "# # \"writesubtitles\": False,\n", "# # \"allsubtitles\": True,\n", "# # \"subtitlesformat\": \"best\",\n", "# \"listallsubtitles\": True,\n", "# # \"socket_timeout\": 5.0,\n", "# \"socket_timeout\": 5.0,\n", "# \"proxy\": US_PROXY,\n", "# }" ] }, { "cell_type": "code", "execution_count": null, "id": "96877bbf", "metadata": {}, "outputs": [], "source": [ "YDL_OPTS = {\n", " \"format\": \"worstaudio\",\n", " \"outtmpl\": os.path.join(DATA_DIR, \"audio\", \"%(id)s.%(ext)s\"),\n", " \"writesubtitles\": True,\n", " \"allsubtitles\": True,\n", " \"subtitlesformat\": \"best\",\n", " \"socket_timeout\": 5.0,\n", " \"proxy\": US_PROXY,\n", "}\n", "MAX_DOWNLOAD_TIME_S = 60 * 2\n", "\n", "YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n", "\n", "def _resolve_youtube(youtube_id):\n", " url = YOUTUBE_BASE_URL + youtube_id\n", " out_data = {\n", " \"id\": youtube_id,\n", " }\n", " t0 = time.time()\n", " try:\n", " with redirect_stderr(open(os.devnull, \"w\")):\n", " with redirect_stdout(open(os.devnull, \"w\")):\n", " with youtube_dl.YoutubeDL(YDL_OPTS) as ydl:\n", " # pre-download to check if we want it (duration, views etc)\n", "# info = ydl.extract_info(url, download=False)\n", " if MAX_DOWNLOAD_TIME_S is not None:\n", " with time_limit(MAX_DOWNLOAD_TIME_S):\n", " info = ydl.extract_info(url, download=True)\n", " else:\n", " info = ydl.extract_info(url, download=True)\n", " _ = info.pop(\"formats\", None)\n", " _ = info.pop(\"thumbnails\", None)\n", " # TODO: figure out language info somehow\n", " _ = info.pop(\"automatic_captions\", None)\n", " out_data[\"success\"] = True\n", " out_data[\"retry\"] = False\n", " out_data[\"meta\"] = info\n", " out_data[\"audio_filename\"] = f\"{info['id']}.{info['ext']}\"\n", " except Exception as e:\n", " out_data[\"success\"] = False\n", " out_data[\"retry\"] = (\n", " \"TimeoutException\" in str(type(e)) or\n", " \"unable to download video data\" in str(e) or \n", " \"No video formats found\" in str(e)\n", " )\n", " out_data[\"fail_type\"] = str(type(e))\n", " out_data[\"fail_message\"] = str(e)\n", " t1 = time.time()\n", " out_data[\"runtime_s\"] = round(t1 - t0, 1)\n", " return out_data" ] }, { "cell_type": "markdown", "id": "757d606b", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "00147f73", "metadata": {}, "outputs": [], "source": [ "# TLDR: setting language hl helps but otherwise seed terms in language matter the most" ] }, { "cell_type": "code", "execution_count": null, "id": "94e9d4f2", "metadata": {}, "outputs": [], "source": [ "# TODO: see what we need to do to get the highest fraction videos with ONLY german subtitles\n", "# - proxy in country\n", "# - specify location with google\n", "# - sort by upload time" ] }, { "cell_type": "code", "execution_count": 1, "id": "5eae95b4", "metadata": {}, "outputs": [], "source": [ "import requests\n", "import json\n", "import re\n", "import time\n", "import random\n", "import urllib.parse\n", "\n", "def _parse_youtube_search_results(html_str):\n", " start_idx = html_str.index(\"ytInitialData\") + len(\"ytInitialData\") + 3\n", " end_idx = html_str.index(\"};\", start_idx) + 1\n", " json_str = html_str[start_idx:end_idx]\n", " page_info = json.loads(json_str)\n", " primary_content = page_info.get(\"contents\", {}).get(\"twoColumnSearchResultsRenderer\", {}).get(\"primaryContents\")\n", " if primary_content is not None:\n", " content_blocks = primary_content[\"sectionListRenderer\"][\"contents\"]\n", " else:\n", " continuation_content = page_info[\"onResponseReceivedCommands\"][0][\"appendContinuationItemsAction\"]\n", " content_blocks = continuation_content[\"continuationItems\"]\n", " results = []\n", " for contents in content_blocks[:-1]:\n", " for video in contents[\"itemSectionRenderer\"][\"contents\"]:\n", " res = {}\n", " if \"videoRenderer\" in video.keys():\n", " video_data = video.get(\"videoRenderer\", {})\n", " res[\"id\"] = video_data.get(\"videoId\", None)\n", " res[\"title\"] = video_data.get(\"title\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"long_desc\"] = video_data.get(\"descriptionSnippet\", {}).get(\"runs\", [{}])[0].get(\"text\", None)\n", " res[\"channel\"] = video_data.get(\"longBylineText\", {}).get(\"runs\", [[{}]])[0].get(\"text\", None)\n", " res[\"duration\"] = video_data.get(\"lengthText\", {}).get(\"simpleText\", None)\n", " res[\"views\"] = video_data.get(\"viewCountText\", {}).get(\"simpleText\", None)\n", " res[\"publish_time\"] = video_data.get(\"publishedTimeText\", {}).get(\"simpleText\", None)\n", " res[\"url_suffix\"] = video_data.get(\n", " \"navigationEndpoint\", {}\n", " ).get(\"commandMetadata\", {}).get(\"webCommandMetadata\", {}).get(\"url\", None)\n", " results.append(res)\n", " if \"continuationItemRenderer\" not in content_blocks[-1]:\n", " continuation_token = None\n", " else:\n", " continuation_endpoint = content_blocks[-1][\"continuationItemRenderer\"][\"continuationEndpoint\"]\n", " continuation_token = continuation_endpoint[\"continuationCommand\"][\"token\"]\n", " return continuation_token, results\n", "\n", "def _get_youtube_search_results(\n", " query_term, \n", " max_n_pages=None,\n", " sp=\"EgQQASgB\", # videos, subtitles, by relevance\n", " hl=\"en\", \n", " gl=None,\n", " proxy_url=None, \n", "):\n", " # NOTE: this returns max ~500 results\n", " query_term = urllib.parse.quote(query_term)\n", " if proxy_url is not None:\n", " proxies = {\n", " \"http\": proxy_url,\n", " \"https\": proxy_url,\n", " }\n", " else:\n", " proxies = None\n", " # prepare url\n", " base_url = \"https://www.youtube.com/results?\"\n", " urls_param_list = [f\"search_query={query_term}\"]\n", " if sp is not None:\n", " urls_param_list.append(f\"sp={sp}\")\n", " if hl is not None:\n", " urls_param_list.append(f\"hl={hl}\")\n", " if gl is not None:\n", " urls_param_list.append(f\"gl={gl}\")\n", " urls_param_str = \"&\".join(urls_param_list)\n", " search_url = base_url + urls_param_str\n", " # grab result pages\n", " all_results = []\n", " seen_ids = set()\n", " continuation_token = \"\"\n", " n_pages = 1\n", " while True:\n", " if n_pages == 1:\n", " harvest_url = search_url\n", " else:\n", " harvest_url = search_url + f\"&ctoken={continuation_token}&hl=de&sp=EgQQASgB\"\n", " out = requests.get(harvest_url, proxies=proxies)\n", " assert(out.ok)\n", " continuation_token, results = _parse_youtube_search_results(out.text)\n", " for m in results:\n", " if m[\"id\"] in seen_ids:\n", " continue\n", " seen_ids.add(m[\"id\"])\n", " all_results.append(m)\n", " if continuation_token is None:\n", " break\n", " if max_n_pages is not None and n_pages >= max_n_pages:\n", " break\n", " n_pages += 1\n", " time.sleep(0.3)\n", " return all_results" ] }, { "cell_type": "code", "execution_count": 2, "id": "c1ca3bdf", "metadata": {}, "outputs": [], "source": [ "import youtube_dl\n", "from contextlib import redirect_stderr, redirect_stdout\n", "import os\n", "\n", "DE_PROXY_IP = \"194.61.113.135\"\n", "PROXY_STR = (\n", " f\"http://brd-customer-hl_98887cab-zone-data_center-route_err-block-country-de-ip-{DE_PROXY_IP}:\" +\n", " \"ote0iqh3psfm@zproxy.lum-superproxy.io:22225\"\n", ")\n", "\n", "\n", "YDL_OPTS = {\n", "# \"writesubtitles\": False,\n", "# \"allsubtitles\": True,\n", "# \"subtitlesformat\": \"best\",\n", " \"listallsubtitles\": True,\n", "# \"socket_timeout\": 5.0,\n", "# \"proxy\": PROXY_STR,\n", "}\n", "\n", "YOUTUBE_BASE_URL = \"https://www.youtube.com/watch?v=\"\n", "\n", "def get_subtitles(youtube_id):\n", " url = YOUTUBE_BASE_URL + youtube_id\n", " with redirect_stderr(open(os.devnull, \"w\")):\n", " with redirect_stdout(open(os.devnull, \"w\")):\n", " with youtube_dl.YoutubeDL(YDL_OPTS) as ydl:\n", " # pre-download to check if we want it (duration, views etc)\n", " info = ydl.extract_info(url, download=False)\n", " return sorted(list(set(info[\"subtitles\"].keys())))" ] }, { "cell_type": "code", "execution_count": 3, "id": "769d6378", "metadata": {}, "outputs": [], "source": [ "import tqdm\n", "TEST_WORDS = [\n", " \"banane\",\n", " \"kürbis\",\n", " \"schaukelpferd\",\n", " \"christbaum\",\n", " \"blind\",\n", "]\n", "cache_dict = {}" ] }, { "cell_type": "code", "execution_count": 5, "id": "b3edf929", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████| 5/5 [06:40<00:00, 80.09s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "43 115 192 192\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "n_tot = 0\n", "n_has_any = 0\n", "n_has_de = 0\n", "n_only_de = 0\n", "for word in tqdm.tqdm(TEST_WORDS):\n", " search_results = _get_youtube_search_results(\n", " word, \n", " max_n_pages=2, \n", "# proxy_url=PROXY_STR,\n", "# gl=\"DE\",\n", " hl=\"de\",\n", " sp=\"EgQQASgB\",\n", "# sp=\"CAISAigB\", # sort by upload_date\n", " )\n", " for m in search_results:\n", " n_tot += 1\n", " if m[\"id\"] in cache_dict:\n", " subtitle_langs = cache_dict[m[\"id\"]]\n", " else:\n", " subtitle_langs = get_subtitles(m[\"id\"])\n", " cache_dict[m[\"id\"]] = subtitle_langs\n", " if len(subtitle_langs) >= 1:\n", " n_has_any += 1\n", " if \"de\" in subtitle_langs:\n", " n_has_de += 1\n", " if len(subtitle_langs) == 1:\n", " n_only_de += 1\n", "print(n_only_de, n_has_de, n_has_any, n_tot)" ] }, { "cell_type": "code", "execution_count": null, "id": "aebd6f73", "metadata": {}, "outputs": [], "source": [ "# 43 115 192 192" ] }, { "cell_type": "code", "execution_count": null, "id": "818de096", "metadata": {}, "outputs": [], "source": [ "# TODO: test adding hl to page 2" ] }, { "cell_type": "code", "execution_count": null, "id": "a8381d19", "metadata": {}, "outputs": [], "source": [ "# bare case\n", "# 24 95 192 192\n", "\n", "# hl=de\n", "# 41 116 191 191\n", "\n", "# hl=de_DE\n", "# 42 118 193 193\n", "\n", "# hl=de_DE, gl=DE\n", "# 42 123 191 191\n", "\n", "# hl=de_DE, proxy\n", "# 42 105 192 192\n", "\n", "# hl=de_DE, gl=DE, proxy\n", "# 44 113 197 197\n", "\n", "# hl=de_DE, gl=DE, proxy, upload_date\n", "# 44 89 159 159" ] }, { "cell_type": "code", "execution_count": null, "id": "c023c21d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "c2ca042b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1a2b0399", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e0c6ebc3", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }