{ "cells": [ { "cell_type": "code", "execution_count": 3, "id": "7f917a9d", "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "import tqdm" ] }, { "cell_type": "markdown", "id": "a76feb70", "metadata": {}, "source": [ "### get seed terms" ] }, { "cell_type": "code", "execution_count": null, "id": "09fa20df", "metadata": {}, "outputs": [], "source": [ "import gzip\n", "import tempfile\n", "import requests\n", "\n", "SUPPORTED_LANG_CODES = set([\n", " 'af', 'als', 'am', 'an', 'ar', 'arz', 'as', 'ast', 'az', 'azb', 'ba', 'bar', 'bcl', 'be', 'bg', \n", " 'bh', 'bn', 'bo', 'bpy', 'br', 'bs', 'ca', 'ce', 'ceb', 'ckb', 'co', 'cs', 'cv', 'cy', 'da', \n", " 'de', 'diq', 'dv', 'el', 'eml', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'fi', 'fr', 'frr', 'fy', \n", " 'ga', 'gd', 'gl', 'gom', 'gu', 'gv', 'he', 'hi', 'hif', 'hr', 'hsb', 'ht', 'hu', 'hy', 'ia', 'id', \n", " 'ilo', 'io', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'ku', 'ky', 'la', 'lb', 'li', \n", " 'lmo', 'lt', 'lv', 'mai', 'mg', 'mhr', 'min', 'mk', 'ml', 'mn', 'mr', 'mrj', 'ms', 'mt', 'mwl', 'my', \n", " 'myv', 'mzn', 'nah', 'nap', 'nds', 'ne', 'new', 'nl', 'nn', 'no', 'nso', 'oc', 'or', 'os', 'pa', \n", " 'pam', 'pfl', 'pl', 'pms', 'pnb', 'ps', 'pt', 'qu', 'rm', 'ro', 'ru', 'sa', 'sah', 'sc', 'scn', 'sco',\n", " 'sd', 'sh', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'su', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'tk', 'tl', \n", " 'tr', 'tt', 'ug', 'uk', 'ur', 'uz', 'vec', 'vi', 'vls', 'vo', 'wa', 'war', 'xmf', 'yi', 'yo', 'zea', 'zh'\n", "])" ] }, { "cell_type": "markdown", "id": "be7ead9e", "metadata": {}, "source": [ "#### download" ] }, { "cell_type": "code", "execution_count": 1, "id": "b4464800", "metadata": {}, "outputs": [], "source": [ "URL_PTN = \"https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.{lang_code}.300.vec.gz\"\n", "\n", "def get_language_unigrams(lang_code):\n", " if lang_code not in SUPPORTED_LANG_CODES:\n", " raise NotImplementedError(f\"language '{lang_code}' not supported\")\n", " with tempfile.TemporaryDirectory() as temp_dir:\n", " tmp_wv_fp = os.path.join(temp_dir, f\"cc.{lang_code}.300.vec.gz\")\n", " # download\n", " wv_url = URL_PTN.format(lang_code=lang_code)\n", " out = requests.get(wv_url)\n", " with open(tmp_wv_fp, \"wb\") as f:\n", " f.write(out.content)\n", " # parse unigrams\n", " unigrams = []\n", " with gzip.open(tmp_wv_fp) as f:\n", " for n, line in enumerate(f):\n", " if n == 0:\n", " continue\n", " line = line.decode(\"utf8\").strip()\n", " if len(line) == 0:\n", " continue\n", " items = line.split(\" \")\n", " if len(items) > 0:\n", " unigram = items[0].strip()\n", " if len(unigram) == 0:\n", " continue\n", " unigrams.append(items[0])\n", " return unigrams\n", "\n", "DATA_DIR = \"/data2/suno/data/harvest/youtube_ml/query_terms\"\n", "os.makedirs(DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 27, "id": "9d2987bf", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████| 158/158 [1:04:02<00:00, 24.32s/it]\n" ] } ], "source": [ "failed_lang_codes = []\n", "for lang_code in tqdm.tqdm(SUPPORTED_LANG_CODES):\n", " out_fp = os.path.join(DATA_DIR, f\"unigrams.{lang_code}.json\")\n", " if os.path.exists(out_fp):\n", " continue\n", " try:\n", " unigrams = get_language_unigrams(lang_code)\n", " with open(out_fp, \"w\") as f:\n", " json.dump(unigrams, f)\n", " except:\n", " failed_lang_codes.append(lang_code)" ] }, { "cell_type": "markdown", "id": "157cd183", "metadata": {}, "source": [ "#### assemble" ] }, { "cell_type": "code", "execution_count": 147, "id": "65085f24", "metadata": {}, "outputs": [], "source": [ "import sys\n", "import unicodedata\n", "from suno_utils.utils.text import normalize_whitespace, make_unique_list\n", "\n", "PUNCT_REPL_MAP = {\n", " n: ord(\" \") \n", " for n in range(sys.maxunicode) \n", " if unicodedata.category(chr(n))[:1] in set([\"P\", \"S\"])\n", "}\n", "# TODO: maybe also other categories\n", "# https://www.fileformat.info/info/unicode/category/index.htm\n", "\n", "def remove_punctuation(text):\n", " return normalize_whitespace(text.translate(PUNCT_REPL_MAP))" ] }, { "cell_type": "code", "execution_count": 148, "id": "a6936975", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████| 158/158 [13:35<00:00, 5.16s/it]\n" ] } ], "source": [ "import re\n", "import numpy as np\n", "\n", "unigram_dict = {}\n", "for lang_code in tqdm.tqdm(SUPPORTED_LANG_CODES):\n", " unigrams_fp = os.path.join(DATA_DIR, f\"unigrams.{lang_code}.json\")\n", " with open(unigrams_fp) as f:\n", " unigrams = json.load(f)\n", " filtered_unigrams = [remove_punctuation(s) for s in unigrams if s[0] != \"<\"]\n", " filtered_unigrams = [s for s in filtered_unigrams if len(s) > 0 and not re.match(\"[0-9\\s]\", s)]\n", " filtered_unigrams = make_unique_list(filtered_unigrams)\n", " # TODO: set a min/max number of characters?\n", " if len(filtered_unigrams) >= 200_000:\n", " len_arr = np.array([len(s) for s in filtered_unigrams])\n", " lower_limit = np.percentile(len_arr, 10)\n", " upper_limit = np.percentile(len_arr, 90)\n", " tmp_unigrams = [s for s in filtered_unigrams if len(s) >= lower_limit and len(s) <= upper_limit]\n", " if len(tmp_unigrams) >= 200_000:\n", " filtered_unigrams = tmp_unigrams\n", " filtered_unigrams = filtered_unigrams[100:]\n", " unigram_dict[lang_code] = filtered_unigrams[:100_000]" ] }, { "cell_type": "code", "execution_count": 149, "id": "99ea96ea", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████| 158/158 [00:00<00:00, 514918.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "nso - 41039\n", "bo - 73189\n", "hif - 74628\n", "zea - 79051\n", "gv - 83540\n", "ht - 93979\n", "co - 94705\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "for lang_code in tqdm.tqdm(SUPPORTED_LANG_CODES):\n", " if len(unigram_dict[lang_code]) != 100_000:\n", " print(lang_code, \"-\", len(unigram_dict[lang_code]))" ] }, { "cell_type": "code", "execution_count": 151, "id": "d6882d42", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['einmal', 'kommen', 'macht', 'unsere', 'zusammen', 'denen', 'Gemeinde', 'Preis', 'letzten', 'zurück', 'welche', 'München', 'einige', 'deren', 'deutschen', 'Frage', 'worden', 'allen', 'erhalten', 'dafür']\n", "['Roundtable', 'Edling', 'Beheizt', 'Gejammer', 'wappnen', 'Jedi Ritter', 'Spannrolle', 'Wechselbeziehungen', 'Sportbegeisterte', 'Magnaten', 'Vetschau', 'offenkundigen', 'herausgehoben', 'Poppins', 'Wärmeerzeugung', 'Hängeleuchte', 'Przemyśl', 'Vermutet', 'ganztägigen', 'Librairie']\n", "['région', 'projet', 'saison', 'niveau', 'reste', 'bonne', 'ensemble', 'peuvent', 'exemple', 'série', 'souvent', 'centre', 'Après', 'écrit', 'pouvoir', 'mettre', 'général', 'forme', 'début', 'personne']\n", "['行业', '相关', '合作', '看到', '完成', '功能', '能力', '工程', '而且', '同时', '城市', '进入', '希望', '地区', '选择', '一定', '用户', '地方', '情况', '专业']\n" ] } ], "source": [ "print(unigram_dict[\"de\"][:20])\n", "print(unigram_dict[\"de\"][-20:])\n", "print(unigram_dict[\"fr\"][:20])\n", "print(unigram_dict[\"zh\"][:20])" ] }, { "cell_type": "code", "execution_count": null, "id": "522d20c1", "metadata": {}, "outputs": [], "source": [ "# remap language codes\n", "unigram_dict[\"iw\"] = unigram_dict[\"he\"]\n", "unigram_dict[\"fil\"] = unigram_dict[\"tl\"]" ] }, { "cell_type": "code", "execution_count": 150, "id": "e79262ff", "metadata": {}, "outputs": [], "source": [ "with open(os.path.join(DATA_DIR, f\"unigrams.json\"), \"w\") as f:\n", " json.dump(unigram_dict, f)" ] }, { "cell_type": "code", "execution_count": null, "id": "6df369c8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "16e56a99", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5972ce71", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "1a15eb09", "metadata": {}, "source": [ "### Find supported languages" ] }, { "cell_type": "code", "execution_count": 5, "id": "29a95bf7", "metadata": {}, "outputs": [], "source": [ "# seed term langs\n", "with open(\"/data2/suno/data/harvest/youtube_ml/query_terms/unigrams.json\") as f:\n", " seed_terms = json.load(f)\n", "SEED_TERM_LANGS = set(seed_terms.keys())" ] }, { "cell_type": "code", "execution_count": 7, "id": "43b0e6d6", "metadata": {}, "outputs": [], "source": [ "# cc langs\n", "from suno_utils.harvest.youtube.constants.cc import CC_LANGS\n", "CC_LANGS = set([k for k in CC_LANGS.keys() if \"-\" not in k])" ] }, { "cell_type": "code", "execution_count": 18, "id": "3a72cf6c", "metadata": {}, "outputs": [], "source": [ "# text classify langs\n", "from suno_utils.harvest.youtube.constants.text_lang import FASTTEXT_LANGS, BASE_TO_FASTTEXT_REMAP\n", "INV_BASE_TO_FASTTEXT_REMAP = {v: k for k, v in BASE_TO_FASTTEXT_REMAP.items()}\n", "TEXT_CLASS_LANGS = set([INV_BASE_TO_FASTTEXT_REMAP.get(s, s) for s in FASTTEXT_LANGS])" ] }, { "cell_type": "code", "execution_count": 19, "id": "7bfdcc1e", "metadata": {}, "outputs": [], "source": [ "# audio classify langs\n", "from suno_utils.harvest.youtube.constants.audio_lang import SPEECHBRAIN_LANGS, BASE_TO_SPEECHBRAIN_REMAP\n", "INV_BASE_TO_SPEECHBRAIN_REMAP = {v: k for k, v in BASE_TO_SPEECHBRAIN_REMAP.items()}\n", "AUDIO_CLASS_LANGS = set([INV_BASE_TO_SPEECHBRAIN_REMAP.get(s, s) for s, _ in SPEECHBRAIN_LANGS])" ] }, { "cell_type": "code", "execution_count": 33, "id": "7b87a906", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "94 available languages\n", "excluded: {'yi', 'ps', 'ur', 'ar', 'sd', 'iw', 'fa'}\n", "87 supported languages\n" ] } ], "source": [ "# check what we have\n", "from suno_utils.harvest.youtube.constants.base import SUPPORTED_LANGS\n", "\n", "print(len(SEED_TERM_LANGS & CC_LANGS & TEXT_CLASS_LANGS & AUDIO_CLASS_LANGS), \"available languages\")\n", "\n", "assert(\n", " len(\n", " set([s for s, _ in SUPPORTED_LANGS.items()]) -\n", " (SEED_TERM_LANGS & CC_LANGS & TEXT_CLASS_LANGS & AUDIO_CLASS_LANGS)\n", " ) == 0\n", ")\n", "\n", "excluded_set = (\n", " (SEED_TERM_LANGS & CC_LANGS & TEXT_CLASS_LANGS & AUDIO_CLASS_LANGS) - \n", " set([s for s, _ in SUPPORTED_LANGS.items()])\n", ")\n", "print(\"excluded:\", excluded_set)\n", "print(len(SUPPORTED_LANGS), \"supported languages\")" ] }, { "cell_type": "code", "execution_count": null, "id": "16a0af0a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "6a65e8b2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3e24c0f7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "de4aad60", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "416acd03", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "f72963a9", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "code", "execution_count": null, "id": "770c8a1c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8fbd1115", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1c357e39", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "58868d96", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5439fc7b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "24139eb2", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" } }, "nbformat": 4, "nbformat_minor": 5 }