{ "cells": [ { "cell_type": "code", "execution_count": 16, "id": "08218f19", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:29:43.594387Z", "start_time": "2023-12-20T15:29:43.592714Z" } }, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": 17, "id": "76fc0b2c", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:29:44.076978Z", "start_time": "2023-12-20T15:29:44.075133Z" } }, "outputs": [], "source": [ "# vocab: \n", "# 0-60_000 text\n", "# 1x0-3999 semantic\n", "# 8x0-4095 coarse\n", "\n", "# 4000 semantic pad token\n", "# 4001 semantic infer token\n", "# 4096 coarse pad token\n", "# 4097 coarse infer token\n", "# 4098 semantic interleave token\n", "\n", "# Memmaps:\n", "# Nx9x3584 for audio tokens\n", "# Jsons:\n", "# N*Dict with meta keys \n", "# \"dataset\"\n", "# \"original_id\", \"original_duration_s\",\n", "# \"start_s\", \"end_s\", \n", "# \"text_segments\", \"private_text_segments\",\n", "# \"text\", \"private_text\",\n", "# \"tags\", \"private_tags\",\n", "# \"views\",\n", "# Dict with meta keys {\"dataset\": [\"idx_list\"]}\n", "\n", "# Bundles (mert_v2_2x1k & dac_2c_25_8):\n", "# s3://suno-data/datasets/bundles/\n", "# v1/youtube_music\n", "# v1/genius_hq\n", "# v1/jamendo\n", "# v1/imslp\n", "# v1/freesound\n", "# v2/pond5\n", "# v2/deezer\n", "# v2/ytm_tagged" ] }, { "cell_type": "code", "execution_count": 18, "id": "ce43e8ad", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:29:44.769704Z", "start_time": "2023-12-20T15:29:44.767190Z" } }, "outputs": [], "source": [ "%matplotlib inline\n", "from matplotlib import pyplot as plt" ] }, { "cell_type": "code", "execution_count": 4, "id": "5164dde2", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T17:57:38.043590Z", "start_time": "2023-12-16T17:57:38.042114Z" } }, "outputs": [], "source": [ "# TODO: add original_filepath on s3" ] }, { "cell_type": "code", "execution_count": 6, "id": "0e33ff3d", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T17:57:43.116317Z", "start_time": "2023-12-16T17:57:43.103274Z" } }, "outputs": [], "source": [ "import math\n", "import numpy as np\n", "import tqdm\n", "import torch\n", "import funcy\n", "import json\n", "import gc\n", "import re\n", "import random\n", "import tempfile\n", "import collections\n", "from collections import defaultdict\n", "from joblib import Parallel, delayed\n", "from transformers import BertTokenizer\n", "\n", "from suno_utils.utils.text import (\n", " write_jsonl,\n", " read_jsonl,\n", " write_json,\n", " read_json,\n", " normalize_whitespace,\n", ")\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists\n", "\n", "TEXT_CODEBOOK_SIZE = 60_001\n", "TEXT_PAD_TOKEN = TEXT_CODEBOOK_SIZE\n", "TEXT_VOCAB_SIZE = 60_032\n", "\n", "SEMANTIC_CODEBOOK_SIZE = 4000\n", "SEMANTIC_N_CODEBOOKS = 1\n", "SEMANTIC_PAD_TOKEN = SEMANTIC_CODEBOOK_SIZE\n", "SEMANTIC_INFER_TOKEN = SEMANTIC_CODEBOOK_SIZE + 1\n", "SEMANTIC_VOCAB_SIZE = 4032\n", "SEMANTIC_RATE_HZ = 25\n", "SEMANTIC_SHIFT_FACTOR = 50\n", "assert SEMANTIC_VOCAB_SIZE == (np.floor(SEMANTIC_CODEBOOK_SIZE // 64) + 1) * 64\n", "\n", "COARSE_CODEBOOK_SIZE = 4096\n", "COARSE_N_CODEBOOKS = 8\n", "COARSE_PAD_TOKEN = COARSE_CODEBOOK_SIZE\n", "COARSE_INFER_TOKEN = COARSE_CODEBOOK_SIZE + 1\n", "COARSE_VOCAB_SIZE = 4160\n", "COARSE_RATE_HZ = 25\n", "COARSE_SHIFT_FACTOR = 5\n", "assert COARSE_CODEBOOK_SIZE + 3 < COARSE_VOCAB_SIZE\n", "assert COARSE_VOCAB_SIZE % 64 == 0\n", "\n", "assert SEMANTIC_RATE_HZ == COARSE_RATE_HZ\n", "\n", "BLOCK_SIZE = 4288\n", "N_TOKENS_TEXT = 1152\n", "N_TOKENS_AUDIO = 3008 # max 120s of audio\n", "# make sure we have enough space for shift 10\n", "assert BLOCK_SIZE >= (\n", " N_TOKENS_TEXT\n", " + N_TOKENS_AUDIO\n", " + SEMANTIC_N_CODEBOOKS * SEMANTIC_SHIFT_FACTOR\n", " + (COARSE_N_CODEBOOKS - 1) * COARSE_SHIFT_FACTOR\n", ")\n", "\n", "SEMANTIC_EMBED_DIR = \"mert_25_2x4k\"\n", "CODEC_EMBED_DIR = \"dac_2c_25_8\"\n", "\n", "METAS_DIR = \"/home/tony/Work/tony/FineTuning_chirp_v2/metadata/\"\n", "OUT_DATA_DIR = \"/app/suno/data/chirp_v2_finetune_v13\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": 7, "id": "b30bfb77", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T18:02:42.151340Z", "start_time": "2023-12-16T18:00:45.904117Z" } }, "outputs": [], "source": [ "# load manifests of IDs and text and tags etc\n", "meta_info_map = {\n", " # \"genius_hq\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"genius_hq_v4_extra_filter.jsonl\"))},\n", " \"genius_hq\": {\n", " m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"genius_hq_v13.jsonl\"))\n", " },\n", " # \"youtube_music\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"youtube_music.jsonl\"))},\n", " # \"freesound\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"freesound.jsonl\"))},\n", " # \"jamendo\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"jamendo.jsonl\"))},\n", " # \"imslp\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"imslp.jsonl\"))},\n", " \"pond5_music\": {\n", " m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"pond5_music_v12.jsonl\"))\n", " },\n", " # \"deezer\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"deezer.jsonl\"))},\n", " # \"ytm_tagged\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"ytm_tagged.jsonl\"))},\n", " # \"musescore\": {m[\"id\"]: m for m in read_jsonl(os.path.join(METAS_DIR, \"musescore.jsonl\"))},\n", "}" ] }, { "cell_type": "code", "execution_count": 8, "id": "9c043b02", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T18:03:10.172713Z", "start_time": "2023-12-16T18:03:10.129527Z" } }, "outputs": [], "source": [ "def _verify_stuff(dset_name, meta_info):\n", " if dset_name == \"genius_hq\":\n", " assert(\n", " \"private_text_segments\" in meta_info or \n", " \"private_text\" in meta_info or\n", " \"text_segments\" in meta_info or \n", " \"text\" in meta_info\n", " )\n", "\n", "def _trim_to_common(arr_1, arr_2):\n", " common_len = min(len(arr_1), len(arr_2))\n", " arr_1 = arr_1[:common_len]\n", " arr_2 = arr_2[:common_len]\n", " return arr_1, arr_2\n", "\n", "def _parse_arrays(dset_name, meta_info, semantic_arr, coarse_arr):\n", " _verify_stuff(dset_name, meta_info)\n", " # prep segment metas (use semantic for timekeeping)\n", " segments_info = []\n", " # first check if we have known segments\n", " if \"text_segments\" in meta_info:\n", " for m in meta_info[\"text_segments\"]:\n", " segments_info.append((\n", " int(round(m[\"start_s\"]*SEMANTIC_RATE_HZ)), \n", " min(len(semantic_arr), int(round(m[\"end_s\"]*SEMANTIC_RATE_HZ))),\n", " m[\"text\"],\n", " m.get(\"private_text\"),\n", " True,\n", " ))\n", " else:\n", " # randomize offset to not get only multiples if no text available\n", " offs = 0\n", " if \"text\" not in meta_info and random.random() > 0.5:\n", " # don't randomize offsets if we have lyrics\n", " offs = random.randint(1, N_TOKENS_AUDIO-1)\n", " segments_info.append((0, min(len(semantic_arr), offs), None, None, False))\n", " for n in range(int(np.ceil((len(semantic_arr)-offs)/N_TOKENS_AUDIO))):\n", " start_idx = offs + n*N_TOKENS_AUDIO\n", " end_idx = min(len(semantic_arr), offs+(n+1)*N_TOKENS_AUDIO)\n", " if end_idx - start_idx < SEMANTIC_RATE_HZ:\n", " # might as well skip mini ones\n", " continue\n", " segments_info.append((\n", " start_idx, \n", " end_idx,\n", " meta_info.get(\"text\") if n == 0 else None, # add text only to first piece\n", " meta_info.get(\"private_text\") if n == 0 else None, # add text only to first piece\n", " False,\n", " ))\n", " if dset_name == \"musescore\":\n", " # break after first piece only text-audio pairs useful here\n", " # TODO: lang here is a hack\n", " meta_info[\"lang\"] = \"en\"\n", " break\n", " arr_list = []\n", " for sem_start_idx, sem_end_idx, text, private_text, is_aligned in segments_info:\n", " if (\n", " sem_end_idx - sem_start_idx > N_TOKENS_AUDIO or \n", " sem_end_idx - sem_start_idx < SEMANTIC_RATE_HZ # arbitrary\n", " ):\n", " continue\n", " coarse_start_idx = int(round(sem_start_idx * COARSE_RATE_HZ / SEMANTIC_RATE_HZ))\n", " coarse_end_idx = int(round(sem_end_idx * COARSE_RATE_HZ / SEMANTIC_RATE_HZ))\n", " assert(sem_end_idx >= 0 and coarse_start_idx >= 0)\n", " if sem_end_idx > len(semantic_arr) or coarse_end_idx > len(coarse_arr):\n", " continue\n", " # get array segments\n", " arr_s = semantic_arr[sem_start_idx:sem_end_idx,:SEMANTIC_N_CODEBOOKS].copy()\n", " arr_c = coarse_arr[coarse_start_idx:coarse_end_idx,:COARSE_N_CODEBOOKS].copy()\n", " # fix any alignment mistakes\n", " arr_s, arr_c = _trim_to_common(arr_s, arr_c)\n", " assert(len(arr_s) == len(arr_c))\n", " # concat and stack\n", " if len(arr_c) < N_TOKENS_AUDIO:\n", " arr_c = np.pad(\n", " arr_c, \n", " ((0, N_TOKENS_AUDIO-len(arr_c)), (0, 0)),\n", " constant_values=COARSE_PAD_TOKEN, \n", " mode=\"constant\",\n", " )\n", " arr_s = np.pad(\n", " arr_s, \n", " ((0, N_TOKENS_AUDIO-len(arr_s)), (0, 0)),\n", " constant_values=SEMANTIC_PAD_TOKEN, \n", " mode=\"constant\",\n", " )\n", " arr = np.concatenate([arr_s, arr_c], axis=-1)\n", " arr = arr.astype(np.uint16)\n", " assert(arr.shape == (N_TOKENS_AUDIO, SEMANTIC_N_CODEBOOKS + COARSE_N_CODEBOOKS))\n", " new_meta = {\n", " \"id\": meta_info[\"id\"],\n", " \"start_s\": round(sem_start_idx / SEMANTIC_RATE_HZ, 2),\n", " \"end_s\": round(sem_end_idx / SEMANTIC_RATE_HZ, 2),\n", " \"original_duration_s\": round(len(semantic_arr) / SEMANTIC_RATE_HZ, 2),\n", " }\n", " if text is not None:\n", " new_meta[\"text\"] = text\n", " if private_text is not None and private_text != private_text:\n", " new_meta[\"text_private\"] = private_text\n", " new_meta[\"text_lang\"] = meta_info[\"lang\"]\n", " new_meta[\"text_aligned\"] = is_aligned\n", " new_meta[\"dset_suffix\"] = \"lyrics\" if meta_info[\"lang\"] == \"en\" else \"lyrics_foreign\" \n", " if \"tags\" in meta_info:\n", " new_meta[\"tags\"] = meta_info[\"tags\"]\n", " if \"private_tags\" in meta_info:\n", " if \"tags\" in meta_info:\n", " # verify that superset\n", " assert(len(set(meta_info[\"tags\"]) - set(meta_info[\"private_tags\"])) == 0)\n", " if \"tags\" not in meta_info or meta_info[\"private_tags\"] != meta_info[\"tags\"]:\n", " new_meta[\"tags_private\"] = meta_info[\"private_tags\"]\n", " if \"original_id\" in meta_info:\n", " new_meta[\"original_id\"] = meta_info[\"original_id\"]\n", " if \"views\" in meta_info:\n", " new_meta[\"views\"] = meta_info[\"views\"]\n", " arr_list.append((arr, new_meta))\n", " del arr_s, arr_c\n", " return arr_list\n", "\n", "def _process_archives(\n", " dset_name,\n", " s3_semantic_archive_filepaths, \n", " s3_coarse_archive_filepaths,\n", " relevant_metas,\n", "):\n", " semantic_archive = {}\n", " s3_semantic_archive_filepaths = set(s3_semantic_archive_filepaths)\n", " # print(len(s3_semantic_archive_filepaths))\n", " for s3_semantic_archive_filepath in s3_semantic_archive_filepaths:\n", " if not check_s3_file_exists(s3_semantic_archive_filepath):\n", " continue\n", " for k, v in read_from_s3(s3_semantic_archive_filepath, read_f=np.load).items():\n", " semantic_archive[k] = v\n", " coarse_archive = {}\n", " s3_coarse_archive_filepaths = set(s3_coarse_archive_filepaths)\n", " # print(len(s3_coarse_archive_filepaths))\n", " for s3_coarse_archive_filepath in s3_coarse_archive_filepaths:\n", " if not check_s3_file_exists(s3_coarse_archive_filepath):\n", " continue\n", " for k, v in read_from_s3(s3_coarse_archive_filepath, read_f=np.load).items():\n", " coarse_archive[k] = v\n", " semantic_uids, coarse_uids = set(semantic_archive.keys()), set(coarse_archive.keys())\n", " assert(\n", " (len(semantic_uids) < 10 and len(coarse_uids) < 10) or\n", " (len(semantic_uids & coarse_uids) / (len(semantic_uids) + len(coarse_uids)) > 0.1)\n", " )\n", " assert(len(semantic_uids & coarse_uids) > 0)\n", " arr_list = []\n", " for uid in semantic_uids & coarse_uids:\n", " if uid not in relevant_metas:\n", " continue\n", " semantic_arr = semantic_archive[uid]\n", " coarse_arr = coarse_archive[uid]\n", " if np.abs(len(coarse_arr) / COARSE_RATE_HZ - len(semantic_arr) / SEMANTIC_RATE_HZ) > 0.1:\n", " # skip if embeddings not roughly the same duration\n", " continue\n", " semantic_arr, coarse_arr = _trim_to_common(semantic_arr, coarse_arr)\n", " assert(len(coarse_arr) == len(semantic_arr) * COARSE_RATE_HZ / SEMANTIC_RATE_HZ)\n", " arr_list.extend(_parse_arrays(dset_name, relevant_metas[uid], semantic_arr, coarse_arr))\n", " del semantic_archive, coarse_archive\n", " gc.collect()\n", " return arr_list\n", "\n", "def _collect_uids(\n", " s3_semantic_metas_filepaths, \n", " s3_coarse_metas_filepaths,\n", "):\n", " semantic_uids = []\n", " for fp in s3_semantic_metas_filepaths:\n", " semantic_uids.extend([m[\"id\"] for m in read_from_s3(fp, read_f=read_jsonl)])\n", " coarse_uids = []\n", " for fp in s3_coarse_metas_filepaths:\n", " coarse_uids.extend([m[\"id\"] for m in read_from_s3(fp, read_f=read_jsonl)])\n", " return set(semantic_uids) & set(coarse_uids)\n", "\n", "def _prep_data(\n", " dataset,\n", " njobs=5,\n", " chunksize=10,\n", " is_val=False,\n", " n_offs=0,\n", "):\n", " dset_name, dset_version, (start_idx, end_idx), n_sem, n_coarse = dataset\n", " dset_type = \"val\" if is_val else \"tr\"\n", " out_mm_filepath = os.path.join(OUT_DATA_DIR, f\"data_{dset_type}.bin\")\n", " out_metas_filepath = os.path.join(OUT_DATA_DIR, f\"metas_{dset_type}.jsonl\")\n", " tot_duration_dict = defaultdict(int)\n", " n_chunks = int(np.ceil((end_idx - start_idx) / chunksize))\n", " for idx_chunk in tqdm.tqdm(funcy.chunks(chunksize, list(range(start_idx, end_idx))), total=n_chunks):\n", " n_jobs = np.min([njobs, chunksize, len(idx_chunk)])\n", " # collect relevant parts of meta file to avoid copying all to subprocesses\n", " tmp_uid_chunks = Parallel(n_jobs=n_jobs, prefer=\"threads\")(\n", " delayed(_collect_uids)(\n", " [\n", " f\"s3://suno-data/datasets/bundles/{dset_version}/{dset_name}/{SEMANTIC_EMBED_DIR}/\" + \n", " f\"metas/part_{idx_idx}.jsonl\"\n", " for idx_idx in range(idx*n_sem, (idx+1)*n_sem)\n", " ],\n", " [\n", " f\"s3://suno-data/datasets/bundles/{dset_version}/{dset_name}/{CODEC_EMBED_DIR}/\" + \n", " f\"metas/part_{idx_idx}.jsonl\"\n", " for idx_idx in range(idx*n_coarse, (idx+1)*n_coarse)\n", " ],\n", " )\n", " for idx in idx_chunk\n", " ) \n", " uids_per_part = {idx: tmp_uid_chunks[n] for n, idx in enumerate(idx_chunk)}\n", " # print(len(uids_per_part))\n", " # collect data\n", " encoded_arrays_list = Parallel(n_jobs=n_jobs, prefer=\"processes\")(\n", " delayed(_process_archives)(\n", " dset_name,\n", " [\n", " f\"s3://suno-data/datasets/bundles/{dset_version}/{dset_name}/{SEMANTIC_EMBED_DIR}/\" + \n", " f\"part_{idx_idx}.npz\"\n", " for idx_idx in range(idx*n_sem, (idx+1)*n_sem)\n", " ],\n", " [\n", " f\"s3://suno-data/datasets/bundles/{dset_version}/{dset_name}/{CODEC_EMBED_DIR}/\" + \n", " f\"part_{idx_idx}.npz\"\n", " for idx_idx in range(idx*n_coarse, (idx+1)*n_coarse)\n", " ],\n", " {\n", " uid: meta_info_map[dset_name][uid] \n", " for uid in uids_per_part[idx] \n", " if uid in meta_info_map[dset_name]\n", " },\n", " )\n", " for idx in idx_chunk\n", " ) \n", " # print(len(encoded_arrays_list))\n", " add_metas = []\n", " for encoded_arrays in encoded_arrays_list:\n", " # print(len(encoded_arrays))\n", " to_write_len = np.sum([arr.size for arr, _ in encoded_arrays])\n", " if to_write_len == 0:\n", " continue\n", " out_mm = np.memmap(\n", " out_mm_filepath, dtype=np.uint16, mode=\"r+\", shape=(n_offs+to_write_len,)\n", " )\n", " for arr, arr_meta in encoded_arrays:\n", " out_mm[n_offs:n_offs+arr.size] = arr.reshape(-1,)\n", " n_offs += arr.size\n", " dataset_str = dset_name\n", " if \"dset_suffix\" in arr_meta:\n", " dataset_str += f\"_{arr_meta['dset_suffix']}\"\n", " add_meta = {\n", " \"dataset\": dataset_str,\n", " \"id\": arr_meta[\"id\"],\n", " \"start_s\": round(arr_meta[\"start_s\"], 2),\n", " \"end_s\": round(arr_meta[\"end_s\"], 2),\n", " \"original_duration_s\": arr_meta[\"original_duration_s\"],\n", " }\n", " if \"original_id\" in arr_meta:\n", " add_meta[\"original_id\"] = arr_meta[\"original_id\"]\n", " if \"tags\" in arr_meta:\n", " add_meta[\"tags\"] = arr_meta[\"tags\"]\n", " if \"tags_private\" in arr_meta:\n", " add_meta[\"tags_private\"] = arr_meta[\"tags_private\"]\n", " if \"text\" in arr_meta:\n", " add_meta[\"text\"] = arr_meta[\"text\"]\n", " if \"text_private\" in arr_meta:\n", " add_meta[\"text_private\"] = arr_meta[\"text_private\"]\n", " if \"text_lang\" in arr_meta:\n", " add_meta[\"text_lang\"] = arr_meta[\"text_lang\"]\n", " if \"text_aligned\" in arr_meta:\n", " add_meta[\"text_aligned\"] = arr_meta[\"text_aligned\"]\n", " if \"views\" in arr_meta:\n", " add_meta[\"views\"] = arr_meta[\"views\"]\n", " tot_duration_dict[dataset_str] += arr_meta[\"end_s\"] - arr_meta[\"start_s\"]\n", " add_metas.append(add_meta)\n", " # write it once\n", " out_mm.flush()\n", " del out_mm\n", " write_jsonl(\n", " add_metas, \n", " os.path.join(out_metas_filepath), \n", " do_append=bool(n_offs!=0),\n", " )\n", " del encoded_arrays_list\n", " gc.collect()\n", " for k, v in tot_duration_dict.items():\n", " print(f\"{round(v / 60 / 60):,} hours of {k}\")\n", " return n_offs\n", "\n", "def prep_data(\n", " datasets,\n", " is_val=False,\n", " njobs=5,\n", " chunksize=10,\n", "):\n", " n_offs = 0\n", " dset_type = \"val\" if is_val else \"tr\"\n", " out_mm_filepath = os.path.join(OUT_DATA_DIR, f\"data_{dset_type}.bin\")\n", " out_metas_filepath = os.path.join(OUT_DATA_DIR, f\"metas_{dset_type}.jsonl\")\n", " out_info_filepath = os.path.join(OUT_DATA_DIR, f\"info_{dset_type}.json\")\n", " out_mm = np.memmap(out_mm_filepath, dtype=np.uint16, mode=\"w+\", shape=(1,))\n", " with open(out_metas_filepath, \"w\") as f:\n", " f.write(\"\")\n", " print('start prepare data')\n", " for dataset in datasets:\n", " n_offs = _prep_data(\n", " dataset,\n", " njobs=njobs,\n", " chunksize=chunksize,\n", " is_val=is_val,\n", " n_offs=n_offs,\n", " )\n", " datasets_info = {}\n", " with open(out_metas_filepath) as f:\n", " n = 0\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " if m[\"dataset\"] not in datasets_info:\n", " datasets_info[m[\"dataset\"]] = {\"idx_list\": []}\n", " datasets_info[m[\"dataset\"]][\"idx_list\"].append(n)\n", " n += 1\n", " write_json(datasets_info, out_info_filepath)" ] }, { "cell_type": "code", "execution_count": 9, "id": "f862095a", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T18:03:10.668017Z", "start_time": "2023-12-16T18:03:10.665974Z" } }, "outputs": [], "source": [ "NJOBS = 60\n", "CHUNKSIZE = 60" ] }, { "cell_type": "code", "execution_count": 10, "id": "3339fe31", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T18:03:55.149049Z", "start_time": "2023-12-16T18:03:12.522957Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "start prepare data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:28<00:00, 28.67s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "73 hours of genius_hq_lyrics\n", "70 hours of genius_hq_lyrics_foreign\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:13<00:00, 13.80s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "21 hours of pond5_music\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# (start_idx, end_idx), n_archives_semantic, n_archives_coarse\n", "datasets = [\n", " # (\"youtube_music\", \"v1\", (0, 1), 1, 1),\n", " (\"genius_hq\", \"v1\", (0, 10), 1, 1),\n", "# (\"freesound\", \"v1\", (0, 2), 1, 1),\n", "# (\"imslp\", \"v1\", (0, 1), 1, 1),\n", "# (\"jamendo\", \"v1\", (0, 1), 1, 1),\n", " (\"pond5_music\", \"v2\", (0, 10), 1, 1),\n", "# (\"deezer\", \"v2\", (0, 1), 1, 1),\n", "# (\"ytm_tagged\", \"v2\", (0, 1), 1, 1),\n", "# (\"musescore\", \"v2\", (0, 1), 1, 1),\n", "]\n", "prep_data(\n", " datasets,\n", " is_val=True,\n", " njobs=NJOBS,\n", " chunksize=CHUNKSIZE,\n", ")\n", "# 26 hours of youtube_music\n", "# 3 hours of youtube_music_lyrics\n", "# 2 hours of youtube_music_lyrics_foreign\n", "# 4 hours of genius_hq\n", "# 9 hours of genius_hq_lyrics\n", "# 5 hours of genius_hq_lyrics_foreign\n", "# 1 hours of freesound\n", "# 29 hours of imslp\n", "# 36 hours of jamendo\n", "# 14 hours of pond5_music\n", "# 12 hours of deezer\n", "# 7 hours of deezer_lyrics\n", "# 8 hours of deezer_lyrics_foreign\n", "# 33 hours of ytm_tagged\n", "# 3 hours of musescore_lyrics" ] }, { "cell_type": "code", "execution_count": 11, "id": "a2dfa127", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T19:39:52.061520Z", "start_time": "2023-12-16T18:03:55.150558Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "start prepare data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 72/72 [58:12<00:00, 48.50s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "23,095 hours of genius_hq_lyrics_foreign\n", "42,752 hours of genius_hq_lyrics\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 69/69 [36:57<00:00, 32.13s/it]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "8,729 hours of pond5_music\n" ] } ], "source": [ "# (start_idx, end_idx), n_archives_semantic, n_archives_coarse\n", "# don't change the end_idx....X.x\n", "datasets = [\n", "# (\"youtube_music\", \"v1\", (1, 4204), 1, 1),\n", " (\"genius_hq\", \"v1\", (10, 4302), 1, 1),\n", "# (\"freesound\", \"v1\", (2, 1021), 1, 1),\n", "# (\"imslp\", \"v1\", (1, 558), 1, 1),\n", "# (\"jamendo\", \"v1\", (1, 112), 1, 1),\n", " (\"pond5_music\", \"v2\", (10, 4138), 1, 1),\n", "# (\"deezer\", \"v2\", (1, 1538), 1, 1),\n", "# (\"ytm_tagged\", \"v2\", (1, 5545), 1, 1),\n", "# (\"musescore\", \"v2\", (1, 33), 1, 1),\n", "]\n", "prep_data(\n", " datasets,\n", " is_val=False,\n", " njobs=NJOBS,\n", " chunksize=CHUNKSIZE,\n", ")\n", "# youtube_music: ~1.5h runtime\n", "# 113,879 hours of youtube_music\n", "# 12,287 hours of youtube_music_lyrics\n", "# 10,043 hours of youtube_music_lyrics_foreign\n", "# genius_hq: ~1.3h runtime\n", "# 17,615 hours of genius_hq\n", "# 41,544 hours of genius_hq_lyrics\n", "# 17,307 hours of genius_hq_lyrics_foreign\n", "# freesound: ~0.2h runtime\n", "# 410 hours of freesound\n", "# imslp: ~0.2h runtime\n", "# 19,514 hours of imslp\n", "# jamendo: ~0.1h runtime\n", "# 3,726 hours of jamendo\n", "# pond5_music: ~1.1h runtime\n", "# 62,117 hours of pond5_music\n", "# deezer: ~0.5h runtime\n", "# 12,287 hours of deezer\n", "# 10,175 hours of deezer_lyrics\n", "# 6,699 hours of deezer_lyrics_foreign\n", "# ytm_tagged: ~2h runtime\n", "# 152,162 hours of ytm_tagged\n", "# musescore: ~0.1h runtime\n", "# 103 hours of musescore_lyrics" ] }, { "cell_type": "code", "execution_count": 12, "id": "c572bd6f", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T19:39:52.955080Z", "start_time": "2023-12-16T19:39:52.063034Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "201G\t/app/suno/data/chirp_v2_finetune_v13/data_tr.bin\r\n" ] } ], "source": [ "!du -hs /app/suno/data/chirp_v2_finetune_v13/data_tr.bin\n", "# 1003G" ] }, { "cell_type": "code", "execution_count": 13, "id": "999206cd", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T19:39:52.959716Z", "start_time": "2023-12-16T19:39:52.957474Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "DONE!\n" ] } ], "source": [ "print(\"DONE!\")" ] }, { "cell_type": "code", "execution_count": 14, "id": "2ae23b40", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T19:39:53.457731Z", "start_time": "2023-12-16T19:39:52.960852Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3972425\n" ] } ], "source": [ "with open(\"/app/suno/data/chirp_v2_finetune_v13/info_tr.json\", \"r\") as fp:\n", " info_tr = json.load(fp)\n", "n_total = 0\n", "for k in info_tr:\n", " n_total += len(info_tr[k]['idx_list'])\n", "print(n_total)" ] }, { "cell_type": "code", "execution_count": 15, "id": "eb9bed5b", "metadata": { "ExecuteTime": { "end_time": "2023-12-16T19:39:53.801815Z", "start_time": "2023-12-16T19:39:53.459088Z" } }, "outputs": [ { "ename": "NameError", "evalue": "name 'BREAK' is not defined", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mBREAK\u001b[49m\n", "\u001b[0;31mNameError\u001b[0m: name 'BREAK' is not defined" ] } ], "source": [ "BREAK" ] }, { "cell_type": "code", "execution_count": null, "id": "4ad9d1f9", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.119356Z", "start_time": "2023-12-13T06:16:08.119347Z" } }, "outputs": [], "source": [ "print(\"here\")" ] }, { "cell_type": "raw", "id": "0793420f", "metadata": {}, "source": [ "# NEED: tokenizer_60k.json" ] }, { "cell_type": "code", "execution_count": null, "id": "2ab57df8", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.120278Z", "start_time": "2023-12-13T06:16:08.120269Z" } }, "outputs": [], "source": [ "# # verify\n", "# mm = np.memmap(\"/mnt/data/georg/data/chirp_v2/data_val.bin\", dtype=np.uint16, mode=\"r\")\n", "# metas = read_jsonl(\"/mnt/data/georg/data/chirp_v2/metas_val.jsonl\")\n", "# mm = mm.reshape(-1, 3008, 9)\n", "# assert(len(mm) == len(metas))\n", "# assert(mm[:100,:,0].min() >= 0)\n", "# assert(mm[:100,:,0].max() <= 4000)\n", "# assert(mm[:100,:,1:].min() >= 0)\n", "# assert(mm[:100,:,1:].max() <= 4096)" ] }, { "cell_type": "code", "execution_count": null, "id": "ba4ee8dc", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.121033Z", "start_time": "2023-12-13T06:16:08.121024Z" } }, "outputs": [], "source": [ "# !aws s3 cp /mnt/data/georg/data/chirp_v2/info_val.json s3://suno-data/georg/data/chirp_v2/\n", "# !aws s3 cp /mnt/data/georg/data/chirp_v2/metas_val.jsonl s3://suno-data/georg/data/chirp_v2/\n", "# !aws s3 cp /mnt/data/georg/data/chirp_v2/data_val.bin s3://suno-data/georg/data/chirp_v2/\n", "\n", "# !aws s3 cp /mnt/data/georg/data/chirp_v2/info_tr.json s3://suno-data/georg/data/chirp_v2/\n", "# !aws s3 cp /mnt/data/georg/data/chirp_v2/metas_tr.jsonl s3://suno-data/georg/data/chirp_v2/\n", "# !aws s3 cp /mnt/data/georg/data/chirp_v2/data_tr.bin s3://suno-data/georg/data/chirp_v2/" ] }, { "cell_type": "code", "execution_count": null, "id": "f5bdf8ac", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.121872Z", "start_time": "2023-12-13T06:16:08.121863Z" } }, "outputs": [], "source": [ "# !du -hs /mnt/data/georg/data/chirp_v2/data_tr.bin\n", "# # 1003G" ] }, { "cell_type": "code", "execution_count": null, "id": "cbcbca32", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.122577Z", "start_time": "2023-12-13T06:16:08.122568Z" } }, "outputs": [], "source": [ "# # randomly listen to some stuff\n", "# from suno_utils.tasks.dac_2c import preload_models as preload_codec_models\n", "# from suno_utils.tasks.dac_2c import (\n", "# encode as codec_encode,\n", "# decode as codec_decode, \n", "# EMBEDDING_RATE as CODEC_EMBEDDING_RATE,\n", "# )\n", "# _ = preload_codec_models(\"/mnt/data/georg/models/chirp_v2/dac_2c_25x8.pt\")\n", "# mm = np.memmap(\"/mnt/data/georg/data/chirp_v2/data_val.bin\", dtype=np.uint16, mode=\"r\")\n", "# mm = mm.reshape(-1, 3008, 9)\n", "# test_metas = read_jsonl(\"/mnt/data/georg/data/chirp_v2/metas_val.jsonl\")\n", "# test_info = read_json(\"/mnt/data/georg/data/chirp_v2/info_val.json\")\n", "# assert(len(test_metas) == len(mm))\n", "# idx_list = list(range(len(test_metas)))\n", "# #random.shuffle(idx_list)\n", "# #idx_list = [idx for idx in idx_list if \"text\" in test_metas[idx]]\n", "# print(len(mm))" ] }, { "cell_type": "code", "execution_count": null, "id": "ffa828e8", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.123326Z", "start_time": "2023-12-13T06:16:08.123317Z" } }, "outputs": [], "source": [ "# show text and audio\n", "# idx = random.choice(idx_list)\n", "# idx = random.choice(test_info[\"pond5_music\"][\"idx_list\"])\n", "# idx = random.choice(test_info[\"genius_hq_lyrics\"][\"idx_list\"])\n", "# idx = random.choice(test_info[\"musescore_lyrics\"][\"idx_list\"])\n", "# idx = random.choice(test_info[\"deezer_lyrics\"][\"idx_list\"])\n", "# idx = random.choice(test_info[\"youtube_music_lyrics_foreign\"][\"idx_list\"])\n", "# idx_key = random.choice(list(test_info.keys()))\n", "# print(idx_key)\n", "# idx = random.choice(test_info[idx_key][\"idx_list\"])\n", "# assert(\"original_duration_s\" in test_metas[idx])\n", "# print(\"tags:\", test_metas[idx].get(\"tags\"))\n", "# arr = mm[idx,1:].copy().astype(np.int16)[:,1:]\n", "# pad_idx_arr = np.where(arr == 4096)[0]\n", "# if len(pad_idx_arr) > 0:\n", "# arr = arr[:pad_idx_arr[0],:]\n", "# a = codec_decode(arr)\n", "# a.play()\n", "# print(\"text:\", test_metas[idx].get(\"text\"))\n", "# plt.plot(a.array_float[0])" ] }, { "cell_type": "code", "execution_count": null, "id": "7008ddc7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "067a561b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "73c92bda", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "08665ec0", "metadata": {}, "source": [ "## Playground" ] }, { "cell_type": "markdown", "id": "747f0363", "metadata": {}, "source": [ "#### get all tags" ] }, { "cell_type": "code", "execution_count": null, "id": "73286512", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.124047Z", "start_time": "2023-12-13T06:16:08.124039Z" } }, "outputs": [], "source": [ "test_metas = read_jsonl(\"/mnt/data/georg/data/chirp_v2/metas_val.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "a3ed7922", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.124773Z", "start_time": "2023-12-13T06:16:08.124764Z" } }, "outputs": [], "source": [ "tags = []\n", "for e in train_metas:\n", " tags.extend(e.get(\"tags\", []))" ] }, { "cell_type": "code", "execution_count": null, "id": "2eed69a8", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.125508Z", "start_time": "2023-12-13T06:16:08.125499Z" } }, "outputs": [], "source": [ "vc_tags = pd.Series(tags).value_counts()" ] }, { "cell_type": "code", "execution_count": null, "id": "bf1c83e8", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.126090Z", "start_time": "2023-12-13T06:16:08.126082Z" } }, "outputs": [], "source": [ "vc_tags.head(5)" ] }, { "cell_type": "code", "execution_count": null, "id": "f76f1e83", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.126777Z", "start_time": "2023-12-13T06:16:08.126769Z" } }, "outputs": [], "source": [ "vc_tags = vc_tags.to_frame()" ] }, { "cell_type": "code", "execution_count": null, "id": "6766efb1", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.127662Z", "start_time": "2023-12-13T06:16:08.127654Z" } }, "outputs": [], "source": [ "vc_tags.tail()" ] }, { "cell_type": "code", "execution_count": null, "id": "fece8109", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.128387Z", "start_time": "2023-12-13T06:16:08.128379Z" } }, "outputs": [], "source": [ "a = vc_tags[vc_tags[\"count\"]>=3][\"count\"].to_dict()" ] }, { "cell_type": "code", "execution_count": null, "id": "52e1a982", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.128967Z", "start_time": "2023-12-13T06:16:08.128959Z" } }, "outputs": [], "source": [ "test_metas = read_jsonl(\"/mnt/data/georg/data/chirp_v2/metas_val.jsonl\")\n", "test_info = read_json(\"/mnt/data/georg/data/chirp_v2/info_val.json\")" ] }, { "cell_type": "code", "execution_count": null, "id": "b6b9f19e", "metadata": { "ExecuteTime": { "end_time": "2023-12-13T06:16:08.129808Z", "start_time": "2023-12-13T06:16:08.129801Z" } }, "outputs": [], "source": [ "idx = random.choice(test_info[\"musescore_lyrics\"][\"idx_list\"])\n", "print(idx)\n", "m = test_metas[idx]\n", "re.sub(r\"\\[.*?\\]\", \"\", m[\"text\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "5dffab2d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "4531aac2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "835a5690", "metadata": {}, "source": [ "# Check tags" ] }, { "cell_type": "code", "execution_count": 20, "id": "143ac0a7", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:30:08.894434Z", "start_time": "2023-12-20T15:30:08.892028Z" } }, "outputs": [ { "data": { "text/plain": [ "dict_keys(['genius_hq', 'pond5_music'])" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "meta_info_map.keys()" ] }, { "cell_type": "code", "execution_count": 24, "id": "532cd6ba", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:30:58.186094Z", "start_time": "2023-12-20T15:30:58.183873Z" } }, "outputs": [], "source": [ "genius_map = meta_info_map[\"genius_hq\"]\n", "pd5_map = meta_info_map[\"pond5_music\"]" ] }, { "cell_type": "code", "execution_count": 29, "id": "aa861cfe", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:32:56.727898Z", "start_time": "2023-12-20T15:32:54.843898Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████████████████████████████████████████████████████████████████| 1237956/1237956 [00:01<00:00, 658266.97it/s]\n" ] } ], "source": [ "genius_counter = collections.Counter()\n", "for k, genius_v in tqdm.tqdm(genius_map.items()):\n", " for tag in genius_v['tags']:\n", " genius_counter[tag.lower()] += 1" ] }, { "cell_type": "code", "execution_count": 30, "id": "7459eaab", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:33:19.896317Z", "start_time": "2023-12-20T15:33:16.252077Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 282713/282713 [00:03<00:00, 77653.80it/s]\n" ] } ], "source": [ "pd5_counter = collections.Counter()\n", "for k, pd5_v in tqdm.tqdm(pd5_map.items()):\n", " for tag in pd5_v['tags']:\n", " pd5_counter[tag.lower()] += 1" ] }, { "cell_type": "code", "execution_count": 32, "id": "e6cf61ce", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:33:39.622119Z", "start_time": "2023-12-20T15:33:39.619830Z" } }, "outputs": [ { "data": { "text/plain": [ "(553004, 571)" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(pd5_counter), len(genius_counter)" ] }, { "cell_type": "code", "execution_count": 36, "id": "2139d8a2", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:35:55.912668Z", "start_time": "2023-12-20T15:35:55.910265Z" } }, "outputs": [ { "data": { "text/plain": [ "21703" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd5_counter[\"festive\"]" ] }, { "cell_type": "code", "execution_count": 39, "id": "d7d13e84", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:38:48.171699Z", "start_time": "2023-12-20T15:38:48.169101Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "552606\n", "552606\n" ] } ], "source": [ "print(len(pd5_counter))\n", "for k in genius_counter:\n", " if k in pd5_counter:\n", " del pd5_counter[k]\n", "print(len(pd5_counter))" ] }, { "cell_type": "code", "execution_count": 42, "id": "f664a6c5", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:42:29.285141Z", "start_time": "2023-12-20T15:42:28.981066Z" } }, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 43, "id": "8a31846e", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:42:42.203011Z", "start_time": "2023-12-20T15:42:42.182712Z" } }, "outputs": [], "source": [ "tags_df = pd.read_csv(\"/home/tony/Work/tony/audios/chirp_popular_tags.csv\")" ] }, { "cell_type": "code", "execution_count": 52, "id": "a71f93ea", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:47:10.264727Z", "start_time": "2023-12-20T15:47:10.262210Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Index(['Event', 'tags', 'Dec 13 2023, 12:00AM - Dec 20 2023, 10:40AM',\n", " 'counts'],\n", " dtype='object')\n" ] } ], "source": [ "print(tags_df.columns)" ] }, { "cell_type": "code", "execution_count": 50, "id": "817060d8", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:45:23.467502Z", "start_time": "2023-12-20T15:45:23.464970Z" } }, "outputs": [], "source": [ "tags_df[\"counts\"] = tags_df[\"Dec 13 2023, 12:00AM - Dec 20 2023, 10:40AM\"]" ] }, { "cell_type": "code", "execution_count": 58, "id": "df8dcc80", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:51:01.668793Z", "start_time": "2023-12-20T15:51:01.249549Z" } }, "outputs": [], "source": [ "tags_counter = collections.Counter()\n", "for _, row in tags_df.iterrows():\n", " tag = row[\"tags\"]\n", " count = row[\"counts\"]\n", " tag = re.sub(\"^w\", \"\", str(tag).lower().strip())\n", " tags_counter[tag] += count" ] }, { "cell_type": "code", "execution_count": 77, "id": "cc4fa6fa", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T17:22:44.530341Z", "start_time": "2023-12-20T17:22:44.447424Z" } }, "outputs": [ { "data": { "text/plain": [ "[('medium tempo', 165135),\n", " ('positive', 127135),\n", " ('uplifting', 114733),\n", " ('inspirational', 109350),\n", " ('motivational', 106305),\n", " ('energetic', 95510),\n", " ('happy', 93629),\n", " ('bright', 91704),\n", " ('optimistic', 90588),\n", " ('background', 88801)]" ] }, "execution_count": 77, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd5_counter.most_common(10)" ] }, { "cell_type": "code", "execution_count": 70, "id": "2cc4bb5a", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T15:55:33.346605Z", "start_time": "2023-12-20T15:55:33.340419Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "user prompted -- energetic 255469 ---pd5 count: 95510\n", "user prompted -- upbeat 208726 ---pd5 count: 58454\n", "user prompted -- emotional 60638 ---pd5 count: 39745\n", "user prompted -- romantic 56146 ---pd5 count: 43954\n", "user prompted -- uplifting 44374 ---pd5 count: 114733\n", "user prompted -- festive 37887 ---pd5 count: 21703\n", "user prompted -- aggressive 32386 ---pd5 count: 26760\n", "user prompted -- intense 32147 ---pd5 count: 44115\n", "user prompted -- playful 31386 ---pd5 count: 55160\n", "user prompted -- groovy 26817 ---pd5 count: 53668\n", "user prompted -- epic 26739 ---pd5 count: 49031\n", "user prompted -- mellow 25282 ---pd5 count: 37834\n", "user prompted -- sad 23203 ---pd5 count: 21271\n", "user prompted -- dreamy 22622 ---pd5 count: 70757\n", "user prompted -- vocals 19871 ---pd5 count: 6792\n", "user prompted -- fast 19770 ---pd5 count: 8283\n", "user prompted -- nostalgic 19473 ---pd5 count: 10774\n", "user prompted -- guitar 17656 ---pd5 count: 36394\n", "user prompted -- melancholic 16604 ---pd5 count: 27002\n", "user prompted -- sentimental 14865 ---pd5 count: 38725\n", "user prompted -- powerful 14472 ---pd5 count: 70666\n", "user prompted -- catchy 11701 ---pd5 count: 18789\n", "user prompted -- happy 11137 ---pd5 count: 93629\n", "user prompted -- love 10425 ---pd5 count: 21277\n", "user prompted -- 80s 10407 ---pd5 count: 7555\n", "user prompted -- lively 9955 ---pd5 count: 53938\n", "user prompted -- soothing 9727 ---pd5 count: 28561\n", "user prompted -- funky 8925 ---pd5 count: 23413\n", "user prompted -- rhythmic 7933 ---pd5 count: 8530\n", "user prompted -- joyful 7632 ---pd5 count: 68923\n", "user prompted -- 90s 7594 ---pd5 count: 7092\n", "user prompted -- cheerful 7140 ---pd5 count: 23429\n", "user prompted -- ethereal 6982 ---pd5 count: 23817\n", "user prompted -- futuristic 6780 ---pd5 count: 27278\n", "user prompted -- dramatic 6772 ---pd5 count: 58963\n", "user prompted -- haunting 6686 ---pd5 count: 23958\n", "user prompted -- sensual 6137 ---pd5 count: 20866\n", "user prompted -- party 5916 ---pd5 count: 17838\n", "user prompted -- kids 5754 ---pd5 count: 10297\n", "user prompted -- electric 5691 ---pd5 count: 5642\n", "user prompted -- passionate 5647 ---pd5 count: 33550\n", "user prompted -- jazzy 4964 ---pd5 count: 5888\n", "user prompted -- bouncy 4807 ---pd5 count: 50360\n", "user prompted -- energy 4805 ---pd5 count: 17163\n", "user prompted -- motivational 4235 ---pd5 count: 106305\n", "user prompted -- inspirational 4016 ---pd5 count: 109350\n", "user prompted -- strong 3812 ---pd5 count: 10192\n" ] } ], "source": [ "potential_bad_tags = {}\n", "for (tag, count) in tags_counter.most_common(300):\n", " if pd5_counter[tag] > 5000:\n", " for g_tag in genius_counter:\n", " if tag in g_tag:\n", " break\n", " else:\n", " print(\"user prompted --\", tag, count, \"---pd5 count:\", pd5_counter[tag])\n", " potential_bad_tags[tag] = count" ] }, { "cell_type": "code", "execution_count": 71, "id": "b5efe1c3", "metadata": { "ExecuteTime": { "end_time": "2023-12-20T16:01:13.645763Z", "start_time": "2023-12-20T16:01:13.643661Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'energetic': 255469, 'upbeat': 208726, 'emotional': 60638, 'romantic': 56146, 'uplifting': 44374, 'festive': 37887, 'aggressive': 32386, 'intense': 32147, 'playful': 31386, 'groovy': 26817, 'epic': 26739, 'mellow': 25282, 'sad': 23203, 'dreamy': 22622, 'vocals': 19871, 'fast': 19770, 'nostalgic': 19473, 'guitar': 17656, 'melancholic': 16604, 'sentimental': 14865, 'powerful': 14472, 'catchy': 11701, 'happy': 11137, 'love': 10425, '80s': 10407, 'lively': 9955, 'soothing': 9727, 'funky': 8925, 'rhythmic': 7933, 'joyful': 7632, '90s': 7594, 'cheerful': 7140, 'ethereal': 6982, 'futuristic': 6780, 'dramatic': 6772, 'haunting': 6686, 'sensual': 6137, 'party': 5916, 'kids': 5754, 'electric': 5691, 'passionate': 5647, 'jazzy': 4964, 'bouncy': 4807, 'energy': 4805, 'motivational': 4235, 'inspirational': 4016, 'strong': 3812}\n" ] } ], "source": [ "print(potential_bad_tags)" ] }, { "cell_type": "code", "execution_count": 90, "id": "3e2a7b0c", "metadata": { "ExecuteTime": { "end_time": "2023-12-21T16:33:22.658618Z", "start_time": "2023-12-21T16:33:22.570020Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "emotional 39745\n", "this is a powerful cinematic motivational epic trailer with emotional and dynamic orchestral strings, brass, taiko drums and hybrid effects. 34\n", "r&b instrumental. hip hop instrumental. smooth instrumental. emotional instrumental. soulful instrumental. soft instrumental. happy instrumental. love instrumental. carefree instrumental. soft instrumental. 143\n", "emotional music 618\n", "a motivational, dynamic, energizing, inspiring and uplifting emotional business summer pop rock track with piano, acoustic guitars, harmonics, electric guitars, synths, plucks, strings, bass and drums and percussion. this track is suitable for uplifting, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, corporate videos, presentations, promotions, infomercials, social media, tutorials, slideshows, advertising, tv and many more. 27\n", "a melancholic and lyrical royalty free music, full of emotions and sadness. it is great for sad and slow background music, emotional and drama trailers, gentle and quiet moods, thoughtful and sentimental videos, tragic and painful scenes, serious and dark projects, winter videos, love and romantic intro scenes, nature videos, documentary films and much more. orchestration: cello, piano, strings section (cellos, violins, contra basses) 18\n", "a dark and dramatic cinematic soundtrack music, full of suspense and mystery. great for cinematic trailers, tension and thriller videos, action and adventure films , danger and mystery scenes, emotional and psychological drama moods, inspiring soundtrack intro music, chases and runaway scenes, ambient and atmospheric suspense background music and much more. main instruments: cinematic drums and percussion, ambient piano, strings, vocal pads 35\n", "a motivational, ambient, dynamic, inspiring and uplifting emotional business summer pop rock track with synths, acoustic guitars, harmonics, electric guitars, plucks, strings, bass and drums and percussion. this track is suitable for uplifting, technological, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, corporate videos, presentations, promotions, infomercials, social media, tutorials, videos, advertising, tv and many more. 52\n", "calm n gentle acoustic inspirational is a playful & positive background music track. it features harmonics and piano intro, emotional acoustic guitars, grand 102\n", "emotional strings 72\n", "an inspiring, energizing and uplifting emotional ambient business summer pop rock track with piano, acoustic guitars, harmonics, electric guitars, synths, plucks, strings, bass and drums and percussion. this track is suitable for uplifting, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, corporate videos, presentations, promotions, tv and many more. 24\n", "dramatic film score soundtrack with symphonic, orchestral instruments such as strings ensemble, piano, violin, cello, bass, human voice, percussion, choir, woodwinds, harp, flute and background texture pads. it's mixing atmospheric ambient underscore based structure with brilliant melodic elements. mood is dramatic, emotional, soft, lost, touching, sad, passionate, chillout and soundscape. ideal for drama movies, cinematic films, documentaries, inspiring commercials, hollywood movie themes and short film projects. visit our emotional music collection: https://www.pond5.com/collections/495940-emotional 28\n", "a highly emotional cinematic indie rock music loop with a touch of melancholy and anticipation. it evokes feelings of deep contemplation and insight. 19\n", "emotional piano 385\n", "emotional rock 32\n", "an inspiring, beautiful and emotionally fulfilling piece of music perfect for wedding videos, romantic videos, inspirational projects, photo slideshows, motivational presentations, christmas projects, and more. uplifting piano motives, elegant and sweeping violins, emotional swells and tender, sentimental moments combine for a rich, inspirational experience. 46\n", "embark on a cinematic journey of epic proportions with this breathtaking soundtrack. with sweeping orchestral arrangements and breathtaking instrumentation, this music captures the essence of inspiration, grandeur and awe. perfect for film, television or any project that requires a powerful and emotional soundtrack that will leave a lasting impression. 25\n", "emotional film scenes 87\n", "an epic and rousing track for fantasy, adventure, epic trailers. emotional and lively, this track features big percussion, full orchestra, choir, and a live solo soprano vocal. valiant and heroic. 11\n", "traditional japanese instrumental music. very emotional, with asian influences. traditional instruments like shakuhachi, bamboo flute, taiko, shamisen. 11\n", "ambient, cinematic atmosphere - good for exploring and traveling through the dark rpg lands. musical instruments used: mysterious string orchestra powerful brass section dramatic choir icy woodwinds ensemble emotional soloists epic percussion the music inspired by jeremy soule / the elder scrolls universe background music. 19\n", "uplifting, inspiring, modern and positive emotional business technology pop track with synths, harmonics, guitars, plucks, bass and drums and percussion. this track is suitable for modern, uplifting, technological, upbeat, motivational and positive inspiring projects to set an optimistic and happy mood. perfect background for commercials, technology and science projects, lifestyle and travel, medical and health videos, startup promotions, explainer videos, advertising, tv and many more. 39\n", "inspiring cinematic action trailer. inspirational strings, tense and powerful heroic brass, intense underlining drums, emotional piano, background choir - all this motivating orchestra will perfectly complement your project: movie, trailer, youtube, slide show, intro, soundtrack, game, etc. 274\n", "this is a positive and modern acoustic indie rock music, full of inspiration and emotions. it is great for easy listening and radio stations, hopeful and motivational commercials, emotional and nostalgic moods, beautiful and melodic instrumental rock music, wonderful and sentimental intro background, soft and warm acoustic uplifting modern rock music and much more. 18\n", "epic emotional and cinematic modern orchestra - beautiful music with epic orchestral culmination. inspirational, dreamy, hopeful and powerful. very emotional and uplifting orchestra composition, hopeful and with cinematic mood. this original track is perfect for videos, presentations and many more projects. 28\n", "emotional orchestral 32\n", "emotional tv ads 11\n", "emotional guitar 12\n", "emotional scene 51\n", "cinematic emotional piano 29\n", "this is a positive, bright and uplifting background track in corporte style with emotional, energetic, happy, inspiring, motivational and optimistic mood. this song was created for advertising and marketing, business projects, presentations, slideshow, promo and social videos. featuring muted and harmonic guitars, piano, bass, and drums. 36\n", "brooding, introspective piano draws an emotional contour with its dynamic performance. melancholic and moody. 18\n", "epic rock big sound - stadium rock ballad with guitars. suitable for romantic big movie scores with a need for emotional searing big sound sound rock and pop. 12\n", "emotional stress 36\n", "perfect for advertising, timelapse, high tech videos, health, medical projects, presentation, youtube video, technology, inspiring, inspirational, travel, emotional, motivational, atmospheric, uplifting, commercial, electronic, inspiration, minimal, modern, business chill music for corporate presentations, slideshow, product promo, commercial, demonstration, nft, crypto, industrial, mechanics, factory etc. 71\n", "emotional sad 13\n", "piano emotional 33\n", "an inspiring, beautiful and emotionally fulfilling piece of music perfect for inspirational projects, photo slideshows, romantic videos motivational presentations and more. uplifting motives, sentimental moments combine for a rich, inspirational experience. 29\n", "an motivational, dynamic, energizing, inspiring and uplifting emotional business summer pop rock track with piano, acoustic guitars, harmonics, electric guitars, synths, plucks, strings, bass and drums and percussion. this track is suitable for uplifting, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, corporate videos, presentations, promotions, infomercials, social media, tutorials, slideshows, advertising, tv and many more. 12\n", "this royalty free background music track in corporate style with emotional, energetic, happy, inspiring, motivational and optimistic mood! perfect for corporate video, youtube, business presentations, tv commercials, advertising, tv/radio broadcast, slideshow, sport, website, online, film, promo videos and more. perfect for a variety of media projects: technology, lifestyle, medicine, architecture, advertising, inspirational presentations and much more! 11\n", "this is a beautiful and touching piano orchestral piano music, full of love and emotions. great for nostalgic memories and sentimental trailers, sad and emotional scenes, romantic and exciting movie scenes,cinematic and inspirational piano moods, winter and autumn scenes, drama films, warm and hopeful background music, family and wedding background music and much more. 19\n", "emotional underscore 46\n", "emotional underscores 12\n", "this is an emotional and sentimental track orchestrated with piano and symphonic strings. great for romantic and love scenes, slow and soft moods, emotional and drama films, family slideshows and much more. 36\n", "emotional cinematic 122\n", "emotional cello 11\n", "a series of understated and poignant drone soundscapes. emotional and deeply questioning. 15\n", "a motivational, ambient, dynamic, inspiring and uplifting emotional business technology pop rock track with synths, acoustic guitars, harmonics, electric guitars, plucks, strings, bass and drums and percussion. this track is suitable for uplifting, technological, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, corporate videos, presentations, promotions, infomercials, social media, tutorials, videos, advertising, tv and many more. 22\n", "an ambient, dynamic, inspiring and uplifting emotional business technology pop track with synths, harmonics, guitars, plucks, bass and drums and percussion. this track is suitable for uplifting, technological, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, technology and science projects, lifestyle and travel, medical and health videos, startup promotions, explainer videos, advertising, tv and many more. 16\n", "rolling drums and piano draw you in while soaring strings evoke a wistful nostalgia. groovy and emotional, with just the right amount of drive. 18\n", "timelapse background, inspiring ambient, chill cinematic, deep soundscape, atmospheric piano it is atmospheric documentary background music. this documentary music with ambient and emotional mood and atmosphere. this track will be a perfect seasoning for any kind of media production that needs a atmosphere and emotion from emotional atmospheric piano, inspire strings, abstract percussion and atmospheric pad. many thanks and enjoy it. 28\n", "corporate emotional 20\n", "emotional trailer 33\n", "emotional epic 91\n", "epic emotional 92\n", "cinematic emotional 108\n", "emotional background 115\n", "acoustic audio track perfect to set an optimistic mood in your project! suitable for any projects, including corporate presentations, lifestyle and travel, tutorials, family videos, app promos and much more…this track featuring acoustic guitars, emotional piano, percussion and strings. 14\n", "epic – action, power, cinematic background track! dramatic energetic strings, emotional drums and timpanis, heroic and insiring theme, powerful trombons and horns. high quality sound! perfect for party and v 16\n", "positive, inspiring and uplifting emotional business technology travel pop track with synths, harmonics, guitars, plucks, bass and drums and percussion. this track is suitable for uplifting, technological, upbeat, motivational and positive tropical inspiring projects to set an optimistic and happy mood. perfect background for commercials, technology and science projects, lifestyle and travel, medical and health videos, startup promotions, explainer videos, advertising, tv and many more. 18\n", "beautiful, sad, emotional, romantic music with a beautiful melody for piano. can be used as the soundtrack to the film. the track is ideal for sad, dramatic, sorrowful, emotional, romantic, love scenes. 11\n", "sad emotional 23\n", "a highly emotional music track with a touch of melancholy and anticipation. it evokes feelings of deep contemplation and insight. featuring piano, hang drum, udu, percussion and rhodes. it will certainly work well with tv commercials, meditation and yoga projects, slow motion, drone and underwater shots. 19\n", "a powerful, emotional and inspiring epic track with a triumphant and adventurous vibe. perfect for movies, trailers, teasers, intros, video games, time lapses, tutorial videos, sporting events, youtube videos and more! 16\n", "this is a sad and melancholic drama music, full of drama and sorrow. it is great for touching and sentimental background, gentle and slow intro, drama and emotional trailer, ambient sad scenes, thoughtful and soft sad story and much more. 11\n", "emotional and inspiring documentary background music with deep piano chords, ambient pad and strings. sounds calm, soft and cinematic. it is a beautiful story of hope, faith and love, reportage about famous people and historical events. a leisurely story about the secrets of the past, the depths of the night sky, the unexplored corners of the ocean and seas. the past reveals its secrets to us, tells its story. 22\n", "customization is always available - just ask!!! singing dog studios brings you a modern country pop ballad featuring a sweet mandolin up front in the mix that is a true cross-over with nods to country, pop, and bluegrass. it is an emotional track that will pull at heart strings and is the perfect soundtrack or backing track to set just the right mood for your next project. 11\n", "a motivational, ambient, dynamic, inspiring and uplifting emotional business summer pop rock track with piano, acoustic guitars, harmonics, electric guitars, synths, plucks, strings, bass and drums and percussion. this track is suitable for uplifting, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, corporate videos, presentations, promotions, infomercials, social media, tutorials, slideshows, advertising, tv and many more. 32\n", "epic cinematic trailer track perfect to set a heroic mood in your project! this track featuring full symphonic orchestra, emotional piano, powerful brass and strings. suitable for any projects, including trailer, action, adventure videos or movies and much more… 15\n", "valentines day – beautiful, inspiring and emotional cinematic track. atmospheric background ambient music track with a deep sound and romantic mood. perfect for films, trailers, love stories, family album, nature, you tube videos, romantic and sentimental videos, timelapse, wedding and much more videos with nostalgic mood. 25\n", "a highly emotional cinematic indie rock music track with a touch of melancholy and anticipation. it evokes feelings of deep contemplation and insight. 12\n", "inspiring, uplifting and motivational track that builds up into a climax towards the end. very positive, hopeful, thoughtful and emotional instrumentation is with keys, synths, orchestral instruments, bass and drums. dynamic fx and ambient pads are also used to create an even deeper ambient feel to the track. 11\n", "a beautiful chinese style music... perfect for giving the asian flair to your production, film, corporate project, youtube video or any other visual project. it is a soaring track which will pull at your heart strings and envelope you in the beauty and grandeur of china. the flute and strings take you on an emotional journey which makes your heart and soul feel so free. 12\n", "customization is always available - just ask!!! singing dog studios brings you a modern country pop ballad featuring a soaring pedal steel guitar that is a true cross-over with nods to country, pop. it is an emotional track that will pull at heart strings and is the perfect soundtrack or backing track to set just the right mood for your next project. 14\n", "a emotional piano and strings peace for a dramatic or romantic background. 14\n", "this is a beautuful and positive gospel music, full of soul and hopes. great for choir and acaapella music, church and christian scenes, happy and optimistic religious music, retro and vintage music, emotional black music ballads and much more. 12\n", "warm emotional 12\n", "this cinematic orchestral track is composed in the classic rpg music style (dark fantasy, heroic and medieval mood). musical instruments used: aggressive string orchestra powerful brass section dramatic choir driving woodwinds ensemble emotional soloists magical harp epic percussion the soundtrack is the best choice for any video game project requires big, heavy and dark action music background. perfect as a battle, combat or an energetic fighting theme. 11\n", "a powerful, dynamic, inspiring and uplifting emotional business technology pop rock track with synths, harmonics, electric guitars, plucks, bass and drums and percussion. this track is suitable for uplifting, technological, motivationals and inspiring projects to set an optimistic and positive mood. perfect background for commercials, technology and science projects, lifestyle and travel, medical and health videos, startup promotions, explainer videos, advertising, tv and many more. 13\n", "emotional acoustic 11\n" ] } ], "source": [ "# for k in genius_counter:\n", "# print(k, genius_counter[k])\n", "for k in pd5_counter:\n", " if \"emotional\" in k and pd5_counter[k] > 10:\n", " print(k, pd5_counter[k])" ] }, { "cell_type": "code", "execution_count": 92, "id": "7f671ebb", "metadata": { "ExecuteTime": { "end_time": "2023-12-21T19:31:43.816865Z", "start_time": "2023-12-21T19:31:43.814071Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Counter({'rap': 529109, 'pop': 436116, 'rock': 234297, 'r&b': 76485, 'en español': 62517, 'trap': 57622, 'france': 51359, 'deutschland': 49805, 'em português': 37261, 'french rap': 36427, 'россия (russia)': 36033, 'alternative rock': 34710, 'electronic': 34036, 'italy': 32780, 'country': 32689, 'polska': 30148, 'hip-hop': 29130, 'brasil': 28178, 'latin music': 27826, 'alternative': 27532, 'uk': 25040, 'русский рэп (russian rap)': 22959, 'indie rock': 22851, 'polski rap': 21153, 'indie': 19828, 'christian': 18487, 'metal': 18039, 'latin urban': 17266, 'indie pop': 16757, 'singer-songwriter': 16516, 'korean': 15460, 'dance': 15421, 'folk': 15414, 'soul': 15329, 'pop-rock': 15279, 'korea': 14918, 'cover': 14285, 'remix': 13962, 'alternative pop': 13773, 'españa': 13607, 'latin pop': 13305, 'italian rap': 12959, 'soundtrack': 12140, 'underground rap': 12081, 'south korea': 11986, 'scandinavia': 11614, 'punk rock': 11344, 'turkey': 10788, 'metalcore': 10165, 'türkiye': 9980, 'non-music': 9524, 'electro-pop': 9501, 'edm': 9445, 'русский поп (russian pop)': 9234, 'post-hardcore': 9196, 'ballad': 9011, 'uk rap': 8979, 'acoustic': 8875, 'west coast': 8150, 'drill': 8142, 'reggaetón': 8012, 'east coast': 7949, 'synth-pop': 7933, 'spanish rap': 7590, 'japan': 7573, 'pop-punk': 7490, 'japanese': 7367, 'hard rock': 7334, 'spanish urban': 7296, 'emo': 7292, 'nederland': 7264, 'portugal': 7207, 'canada': 7191, 'french pop': 7151, 'türkçe sözlü rap': 7073, 'latin trap': 6556, 'israeli music': 6484, 'dance-pop': 6468, 'dancehall': 6455, 'sverige': 6390, 'folk rock': 6373, 'atlanta rap': 6325, 'australia': 6241, 'jazz': 6222, 'deutschsprachiger pop': 6197, 'heavy metal': 6126, 'argentina': 6051, 'reggae': 5956, 'musicals': 5947, 'русский рок (russian rock)': 5920, 'soul pop': 5901, 'gangsta rap': 5863, 'puerto rico': 5855, 'arabia | عربي': 5719, 'alternative metal': 5601, 'k-pop (케이팝)': 5498, 'funk': 5479, 'emo rap': 5461, 'hardcore rap': 5458, 'live': 5445, 'italian pop': 5346, 'j-pop': 5199, 'jamaica': 5182, 'danmark': 5148, 'house': 4995, 'boy band': 4911, 'românia': 4890, 'русский трэп (russian trap)': 4869, 'k-solo': 4813, 'alternative r&b': 4667, 'arabic rap | راب عربي': 4655, 'british rock': 4600, 'méxico': 4519, 'experimental': 4484, 'cloud rap': 4468, 'lo-fi': 4394, 'eighties': 4261, 'post-punk': 4169, 'christmas': 4167, 'hardcore punk': 4150, 'new wave': 4103, 'progressive rock': 4094, 'conscious hip-hop': 4070, 'srbija': 3927, 'freestyle': 3903, 'comedy': 3862, 'spanish pop': 3856, 'south africa': 3776, 'norge': 3671, 'unreleased': 3594, 'death metal': 3593, 'dream pop': 3524, 'girl group': 3502, 'seventies': 3486, 'psychedelic': 3483, 'psychedelic rock': 3440, 'boom bap': 3417, 'gospel': 3394, 'electronic rock': 3374, 'piano': 3359, 'gaming': 3339, 'deutschsprachiger rock': 3298, 'colombia': 3288, 'art rock': 3260, 'blues': 3249, 'christian rap': 3191, 'progressive metal': 3115, 'hyperpop': 3050, 'afrobeats': 3049, 'nu-metal': 2894, 'indie rap': 2882, 'blues rock': 2834, 'dubstep': 2791, 'israeli rap': 2686, \"children's music\": 2685, 'thrash metal': 2640, 'pop rap': 2631, 'experimental rock': 2576, 'christian pop': 2566, 'funk nacional': 2554, 'svensk rap': 2549, 'disney': 2544, 'tv': 2523, 'nineties': 2503, 'youtube': 2481, 'україна (ukraine)': 2477, 'québec': 2474, 'power metal': 2457, 'österreich': 2422, 'belgië/belgique': 2418, 'uk drill': 2362, 'art pop': 2315, 'neo soul': 2298, 'electro house': 2286, 'romanized': 2283, 'ireland': 2273, 'melodic death metal': 2243, 'anime': 2236, 'suomi': 2234, 'electro': 2181, 'filipino': 2170, 'future bass': 2167, 'adult alternative': 2138, 'morocco | مغربي': 2124, 'nigeria': 2110, 'moroccan rap | راب مغربي': 2101, 'teen pop': 2097, 'norsk': 2085, 'k-r&b': 2081, 'dominican republic': 2053, 'bedroom pop': 2045, 'dirty south': 2026, 'chile': 2025, 'industrial': 2018, 'experimental rap': 2016, 'spoken word': 2013, 'horrorcore': 2008, 'shoegaze': 2007, 'ambient': 1986, 'folk pop': 1975, 'nerdcore': 1969, 'symphonic metal': 1940, 'worship': 1930, 'melodic hardcore': 1926, 'israeli rock': 1924, 'deathcore': 1908, 'rap-rock': 1905, 'new york': 1903, 'vietnam': 1902, 'parody': 1899, 'french r&b': 1843, 'grime': 1833, 'adult contemporary': 1829, 'srpski rap': 1825, 'aussie hip-hop': 1820, 'drum & bass': 1820, 'electronica': 1804, 'israeli pop': 1793, 'country pop': 1787, 'screen': 1777, 'iran': 1776, 'noise rock': 1765, 'persian': 1760, 'egyptian | مصري': 1754, 'memes': 1748, 'disco': 1748, 'demo': 1744, 'dmv': 1736, 'hardcore': 1735, 'grunge': 1727, 'holiday': 1724, 'uruguay': 1711, 'lgbtq+': 1708, 'latin rock': 1689, 'farsi': 1684, 'k-hip-hop': 1652, 'americana': 1647, 'chicago drill': 1637, 'melodic metalcore': 1625, 'soft rock': 1621, 'j-rap': 1614, 'orchestral': 1613, 'chill': 1611, 'neo-psychedelia': 1588, 'j-rock': 1579, 'post-grunge': 1577, 'albania': 1573, 'american underground': 1572, 'québec rap': 1572, 'c-pop': 1566, 'contemporary folk': 1564, 'trap metal': 1563, 'industrial metal': 1562, 'uk r&b': 1562, 'regional mexicano': 1553, 'bay area': 1549, 'garage rock': 1545, 'christian rock': 1514, 'jamaican patois': 1508, 'theme song': 1492, 'power pop': 1491, 'gothic rock': 1479, 'deep house': 1466, 'scandipop': 1464, 'egyptian rap | راب مصري': 1450, 'groove metal': 1450, 'venezuela': 1446, 'broadway': 1428, 'country rap': 1419, 'schlager': 1418, 'electronic trap': 1415, 'literature': 1408, 'latin rap': 1401, 'black metal': 1386, 'motown': 1379, 'funk rock': 1371, 'k-ballad': 1366, 'post-rock': 1363, 'switzerland': 1361, 'progressive house': 1336, 'glitch': 1317, 'hrvatska': 1315, 'meme rap': 1302, 'alternative country': 1289, 'trip-hop': 1284, 'experimental pop': 1274, 'synth rock': 1268, 'k-ost': 1264, 'iranian rap': 1261, 'česko': 1259, 'soul jazz': 1242, 'florida rap': 1239, 'český rap': 1238, 'variété française': 1223, 'farsi rap': 1222, 'synthwave': 1209, 'русское аренби (russian r&b)': 1201, 'intro': 1186, 'poetry': 1176, 'industrial hip-hop': 1165, 'jazz rap': 1163, 'extreme metal': 1157, 'bass music': 1154, 'dark pop': 1154, 'glitch hop': 1151, 'baroque pop': 1147, 'industrial rock': 1142, 'alternative dance': 1134, 'plugg': 1119, 'funk-pop': 1114, 'en français': 1099, 'glam rock': 1076, 'dark wave': 1070, 'mashup': 1065, 'sixties': 1060, 'trance': 1059, 'britpop': 1057, 'techno': 1044, 'desi hip-hop': 1043, 'corrido': 1031, 'downtempo': 1025, 'djent': 1022, 'skramz': 1011, 'русский андеграунд (russian underground)': 1003, 'svensk pop': 995, 'easy listening': 991, 'hip-hop tuga': 983, 'china': 980, 'norsk rap': 979, 'srpski pop': 975, 'chicago rap': 964, 'diss': 958, 'battle rap': 958, 'hong kong': 953, 'religion': 947, 'беларусь (belarus)': 935, 'protest songs': 932, 'g-funk': 926, 'melodic dubstep': 925, 'avant garde': 925, 'sertanejo': 918, 'doom metal': 908, 'instrumental': 888, 'progressive metalcore': 885, 'gothic metal': 878, 'bluegrass': 878, 'mpb (música popular brasileira)': 870, 'svensk rock': 866, 'french rock': 866, 'bubblegum pop': 863, 'a cappella': 860, 'mandopop': 859, 'cumbia': 859, 'eurovision': 856, 'polski rock': 856, 'math rock': 851, 'magyar': 836, 'memphis': 827, 'chapter one': 826, 'ska': 825, 'middle east': 817, 'new york drill': 814, 'jazz fusion': 812, 'русский ютуб (russian youtube)': 805, 'interlude': 804, 'indonesia': 797, 'noise pop': 795, 'israeli yam tichoni': 781, 'bosna': 773, 'american folk': 767, 'polski pop': 764, 'traditional': 761, 'future house': 758, 'rockabilly': 756, 'cantopop': 755, 'india': 739, 'abstract rap': 730, 'speed metal': 725, 'vbt (videobattleturnier)': 724, 'vocaloid': 718, 'folk punk': 711, 'chillout': 697, 'sludge metal': 693, 'politics': 692, 'skate punk': 689, 'indie folk': 684, 'azerbaijan': 683, 'new zealand': 682, 'nu disco': 680, 'slowcore': 677, 'angola': 674, 'k-rock': 658, 'k-indie': 654, 'rage': 648, 'tropical house': 646, 'chamber music': 639, 'balada': 635, 'fantasy (lit)': 632, 'afro trap': 628, 'electro-funk': 624, 'cuba': 624, 'perú': 622, 'қазақстан (kazakhstan)': 621, 'israeli hip-hop': 618, 'noise': 615, 'traditional folk': 604, 'garage punk': 600, 'outro': 594, 'lyrical poetry': 591, 'post-punk revival': 590, 'satire': 589, 'folk metal': 583, 'algerian rap | راب جزائري': 577, 'detroit rap': 576, 'hard dance': 575, 'dembow': 566, 'crioulo cabo-verdiano': 564, 'ska punk': 564, 'skit': 563, 'new jack swing': 555, 'cartoon': 554, 'k-japanese': 553, 'electro-hop': 541, 'rap metal': 533, 'samba': 530, 'electronicore': 529, 'mathcore': 528, 'русский панк-рок (russian punk rock)': 526, 'southern rock': 512, 'stoner rock': 509, 'spanish rock': 502, 'afroswing': 497, 'emo pop': 497, 'symphonic power metal': 493, 'русский хайперпоп (russian hyperpop)': 483, 'k-idol-hip-hop': 480, 'indie electronic': 479, 'deutschsprachiger trap': 478, 'retro': 475, 'scotland': 472, 'punjabi': 469, 'swing': 468, 'dark ambient': 467, 'afrobeat': 466, 'trapwave': 462, 'dark trap': 460, 'vtuber': 458, 'christian metal': 457, 'greece': 456, 'album-oriented rock (aor)': 447, 'chillhop': 446, 'phonk': 438, 'space rock': 434, 'bachata': 427, 'afrika': 426, 'eurodance': 424, 'hindi': 423, 'british folk': 421, 'classical music': 418, 'salsa': 417, '\\u200b\\u200bisizulu': 415, 'doo-wop': 415, 'azerbaijani rap': 409, 'news': 408, 'tech': 400, 'history': 399, 'tunisian rap | راب تونسي': 398, 'neofolk': 392, 'cinematic': 391, 'philosophy': 389, 'african languages': 388, 'lounge': 383, 'big band': 380, 'bossa nova': 379, 'iranian pop': 377, 'experimental folk': 373, 'technical death metal': 373, 'grindcore': 366, 'taiwan': 365, 'roots': 362, 'dub': 362, 'hardstyle': 354, 'psychedelic soul': 353, 'brutal death metal': 353, 'breakbeat': 348, 'christian r&b': 347, 'flamenco': 347, 'vaporwave': 343, 'indian pop': 338, 'planète rap': 336, 'stoner metal': 330, 'drumstep': 329, 'concert': 318, 'theatre': 318, 'post-metal': 317, 'halloween': 314, 'bollywood': 312, 'idm': 311, 'spotify singles': 305, 'jazz-funk': 305, 'chinese hip-hop': 300, 'pluggnb': 299, 'rapcore': 296, 'netflix': 296, 'sports': 291, 'hard bass': 290, 'soul rap': 289, 'hymn': 286, 'folklore': 283, 'chillstep': 282, 'new age': 281, 'synth punk': 280, 'riddim': 269, 'forró': 268, 'norsk rock': 267, 'avant garde metal': 257, 'русский баттл-рэп (russian battle rap)': 252, 'surf punk': 251, 'witch house': 246, 'space': 242, 'crossover thrash': 235, 'traducción al español': 233, 'banda': 233, 'nigerian rap': 232, 'smooth jazz': 231, 'celtic': 231, 'northern ireland': 227, 'underground soul': 215, 'finnish rock': 211, 'irish folk': 207, 'progressive death metal': 207, 'drone': 203, 'wrestling': 195, 'chiptune': 192, 'modern classical': 192, 'blackened death metal': 182, 'horror punk': 179, 'football (soccer)': 177, 'crust punk': 177, 'midwest emo': 172, 'surf rock': 168, 'surf': 159, 'krautrock': 137, 'tradução em português': 136, 'amapiano': 129, 'traduction française': 116, 'latin jazz': 115, 'myth and legend': 106, 'art': 105, 'diy punk': 104, 'gabber': 97, 'nu-jazz': 95, 'contemporary poetry': 94, 'nepali': 87, 'atmospheric black metal': 86, 'italia': 83, 'translation': 82, 'traduzione italiana': 71, 'harsh noise': 67, 'english translation': 66, 'polskie tłumaczenie (polish translation)': 65, 'nintendo': 63, 'türkçe çeviri': 55, 'french literature': 49, 'covid-19': 47, 'tracklist + album art': 40, 'deutsche übersetzung': 38, 'meta': 23, 'русский перевод (russian translation)': 14, 'law': 13, 'list': 9})\n" ] } ], "source": [ "print(genius_counter)" ] }, { "cell_type": "code", "execution_count": null, "id": "537fcf84", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }