{ "cells": [ { "cell_type": "markdown", "id": "2e6bb3a6", "metadata": {}, "source": [ "## 48kHz Music stereo files" ] }, { "cell_type": "code", "execution_count": null, "id": "f9b3eaeb", "metadata": {}, "outputs": [], "source": [ "# ~5Tb for 6k hours\n", "# ~30Tb for 35k hours" ] }, { "cell_type": "code", "execution_count": null, "id": "d72d13d7", "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "from matplotlib import pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "id": "39ed2b9a", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "682bfd7c", "metadata": {}, "outputs": [], "source": [ "import random\n", "import json\n", "import numpy as np\n", "import tqdm\n", "import torch\n", "import funcy\n", "import time\n", "import gc\n", "import tempfile\n", "import collections\n", "from joblib import Parallel, delayed\n", "\n", "from suno_utils.audio import Audio\n", "from suno_utils.tasks.data_loader import load_audio_mp\n", "from suno_utils.utils.text import write_jsonl, read_jsonl, write_json, read_json\n", "from suno_utils.utils.s3 import read_from_s3, check_s3_file_exists, open_from_s3\n", "\n", "SAMPLE_RATE = 48_000\n", "\n", "OUT_DATA_DIR = \"/app/suno/data/audio_2ch_48khz_lg\"\n", "os.makedirs(OUT_DATA_DIR, exist_ok=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "b5fc2a9b", "metadata": {}, "outputs": [], "source": [ "# !rm -rf /app/suno/data/audio_2ch_48khz_lg/*" ] }, { "cell_type": "code", "execution_count": null, "id": "f27344be", "metadata": {}, "outputs": [], "source": [ "def _convert_float_audio(sig):\n", " dtype = np.int16\n", " dtype_info = np.iinfo(dtype)\n", " abs_max = 2 ** (dtype_info.bits - 1)\n", " offset = dtype_info.min + abs_max\n", " return (sig * abs_max + offset).clip(dtype_info.min, dtype_info.max).astype(dtype)\n", "\n", "def load_metas(filepath, simple=True):\n", " assert(filepath.startswith(\"s3://\"))\n", " data = []\n", " with open_from_s3(filepath) as f:\n", " for line in f:\n", " line = line.strip()\n", " if len(line) == 0:\n", " continue\n", " m = json.loads(line)\n", " _id = m[\"id\"]\n", "# duration_s = m[\"duration_s\"]\n", " filepath = m.get(\"s3_filepath\", m.get(\"audio_filepath\", m.get(\"filepath\")))\n", " assert(filepath is not None)\n", " m_new = {\n", " \"id\": _id,\n", " \"filepath\": filepath,\n", "# \"duration_s\": duration_s,\n", " }\n", " if not simple:\n", " for k in m.keys():\n", "# if k not in [\"id\", \"duration_s\", \"s3_filepath\", \"audio_filepath\", \"filepath\"]:\n", " if k not in [\"id\", \"s3_filepath\", \"audio_filepath\", \"filepath\"]:\n", " m_new[k] = m[k]\n", " data.append(m_new)\n", " return data" ] }, { "cell_type": "code", "execution_count": null, "id": "7b8a3eee", "metadata": {}, "outputs": [], "source": [ "## previously\n", "# metas_info = [\n", "# (\"v1/genius_hq\", 1_000),\n", "# (\"v1/youtube_music\", 2_500),\n", "# (\"v1/jamendo\", 200),\n", "# (\"v1/imslp\", 500),\n", "# (\"v1/fma\", 200),\n", "# (\"v2/pond5_music\", 1_000),\n", "# (\"v2/deezer\", 200),\n", "# (\"v2/ytm_tagged\", 200),\n", "# ]" ] }, { "cell_type": "code", "execution_count": null, "id": "58ed6b92", "metadata": {}, "outputs": [], "source": [ "base_s3_dir = \"s3://suno-data/datasets\"\n", "\n", "IS_TEST = False\n", "\n", "# dset_name, metas_filepath, (min_s, max_s), max_hours\n", "raw_metas_info = [\n", "\n", " # speech\n", " (\"podcasts\", \"bundles/v0/podcasts/metas.jsonl\", (60, 2*60), 2_500),\n", "\n", " # music`\n", " (\"genius_hq\", \"bundles/v1/genius_hq/metas.jsonl\", (60, 2*60), 5_000),\n", " (\"youtube_music\", \"bundles/v1/youtube_music/metas.jsonl\", (60, 2*60), 10_000),\n", " (\"jamendo\", \"bundles/v1/jamendo/metas.jsonl\", (60, 2*60), 1_000),\n", " (\"imslp\", \"bundles/v1/imslp/metas.jsonl\", (60, 2*60), 1_000),\n", " (\"pond5_music\", \"bundles/v2/pond5_music/metas.jsonl\", (60, 2*60), 2_500),\n", " (\"spot_genres\", \"harvest/spotify/ytm_spotify_genres_50_simple.jsonl\", (60, 2*60), 10_000),\n", " (\"tency\", \"harvest/tency/tency_plus_fp_flat.jsonl\", (20, 2*60), 500),\n", " (\"shutter_music\", \"bundles/v3/shutter_music/metas_stems_flat.jsonl\", (20, 2*60), 500),\n", " \n", " # misc\n", " (\"pond5_sfx\", \"bundles/v3/pond5_sfx/metas.jsonl\", (5, 30), 500),\n", "\n", "]\n", "\n", "random.seed(6006)\n", "filepath_info = {}\n", "for dset_key, rel_fp, _, _ in tqdm.tqdm(raw_metas_info):\n", " metas = load_metas(os.path.join(base_s3_dir, rel_fp))\n", " random.shuffle(metas)\n", " filepath_info[dset_key] = metas\n", " \n", "if IS_TEST:\n", " metas_info = [(a, b, c, 1) for a, b, c, d in raw_metas_info]\n", "else:\n", " metas_info = [e for e in raw_metas_info]" ] }, { "cell_type": "code", "execution_count": null, "id": "7a2750c6", "metadata": {}, "outputs": [], "source": [ "from joblib import Parallel, delayed\n", "from joblib.externals.loky import get_reusable_executor\n", "from scipy.io import wavfile\n", "\n", "def _write_item(work_item):\n", " audio_arr, out_filepath = work_item\n", " try:\n", " wavfile.write(out_filepath, SAMPLE_RATE, audio_arr.T)\n", " except:\n", " return False\n", " return True\n", "\n", "def _mp_write(audio_arr_list, out_filepaths, num_workers=16):\n", " assert(len(audio_arr_list) == len(out_filepaths))\n", " work_items = list(zip(audio_arr_list, out_filepaths))\n", " confirmed_list = Parallel(n_jobs=num_workers, prefer=\"threads\", batch_size=1)(\n", " delayed(_write_item)(work_item) for work_item in work_items\n", " )\n", " get_reusable_executor().shutdown(wait=True)\n", " return confirmed_list" ] }, { "cell_type": "code", "execution_count": null, "id": "1b253b7b", "metadata": {}, "outputs": [], "source": [ "import uuid\n", "\n", "sample_metas_tr = []\n", "sample_metas_val = []\n", "is_finished = False\n", "# for each dset, load in chunks of 1k files, and keep doing until we have enough\n", "for dset_key, _, (req_min_s, req_max_s), req_duration_h in metas_info:\n", " dset_duration_h = 0\n", " for n_iter, filepaths_chunk in enumerate(funcy.chunks(5000, filepath_info[dset_key])):\n", " is_val = n_iter == 0\n", " if is_val: # val, so make it smaller\n", " filepaths_chunk = filepaths_chunk[:500]\n", " out_dir = os.path.join(OUT_DATA_DIR, \"val\" if is_val else \"train\", dset_key)\n", " os.makedirs(out_dir, exist_ok=True)\n", " filepaths_chunk_list = [m[\"filepath\"] for m in filepaths_chunk]\n", " t0 = time.time()\n", " audio_arr_list = load_audio_mp(\n", " filepaths_chunk_list,\n", " target_sample_rate=SAMPLE_RATE,\n", " n_channels=2,\n", "# min_duration_s=req_min_s, # TODO: not here cause we want to skip\n", " max_duration_s=req_max_s,\n", " normalize_volume=True,\n", " num_workers=32,\n", " force_threads=False,\n", " # debug=False,\n", " silent=True,\n", " )\n", " # add random offset just incase\n", " filtered_audio_arr_list = []\n", " offset_list = []\n", " for arr in audio_arr_list:\n", " if arr is None:\n", " filtered_audio_arr_list.append(arr)\n", " offset_list.append(0)\n", " continue\n", " offset = int(round(random.uniform(0, arr.shape[-1]//4/SAMPLE_RATE), 1)*SAMPLE_RATE)\n", " filtered_audio_arr_list.append(arr[:,offset:])\n", " offset_list.append(offset/SAMPLE_RATE)\n", " audio_arr_list = filtered_audio_arr_list\n", " del filtered_audio_arr_list\n", " audio_arr_list = [\n", " _convert_float_audio(arr.numpy()) if arr is not None else None for arr in audio_arr_list\n", " ]\n", " td_fetch = int(round(time.time() - t0))\n", " time.sleep(5) # make sure things close\n", " # multicore writing\n", " t0 = time.time()\n", " n_offset = len(sample_metas_val) if is_val else len(sample_metas_tr)\n", " new_ids = [str(uuid.uuid4()) for _ in filepaths_chunk]\n", " out_filepaths = [os.path.join(out_dir, f\"{new_id}.wav\") for new_id in new_ids]\n", " confirmed_list = _mp_write(audio_arr_list, out_filepaths)\n", " tot_duration_s = 0\n", " for is_confirmed, new_id, fp, m, offset_s, arr in zip(\n", " confirmed_list, new_ids, out_filepaths, filepaths_chunk, offset_list, audio_arr_list\n", " ):\n", " if not is_confirmed:\n", " continue\n", " duration_s = arr.shape[-1] / SAMPLE_RATE\n", " if duration_s < req_min_s:\n", " continue\n", " new_m = {\n", " \"dataset\": dset_key,\n", " \"id\": new_id,\n", " \"original_id\": m[\"id\"],\n", " \"filepath\": fp,\n", " \"offset_s\": offset_s,\n", " \"duration_s\": round(duration_s, 2),\n", " }\n", " if is_val:\n", " sample_metas_val.append(new_m)\n", " else:\n", " sample_metas_tr.append(new_m)\n", " tot_duration_s += duration_s\n", " chunk_duration_h = round(tot_duration_s / 60 / 60, 1)\n", " del audio_arr_list\n", " gc.collect();\n", " td_write = int(round(time.time() - t0))\n", " time.sleep(5) # make sure things close\n", " dset_type = \"val\" if is_val else \"train\"\n", " print(\n", " f\"{dset_key}: {chunk_duration_h:,} hours of data fetched in {td_fetch:,}s\"\n", " f\" and written in {td_write:,}s as `{dset_type}`, retained {np.mean(confirmed_list)*100:.1f}%\"\n", " )\n", " if not is_val:\n", " dset_duration_h += chunk_duration_h\n", " if dset_duration_h >= req_duration_h:\n", " print(f\"done with {dset_key}, collected total of {dset_duration_h:,.1f} hours for train\")\n", " break \n", "is_finished = True" ] }, { "cell_type": "code", "execution_count": null, "id": "c4be2777", "metadata": {}, "outputs": [], "source": [ "# du -hs /app/suno/data/audio_2ch_48khz_lg" ] }, { "cell_type": "code", "execution_count": null, "id": "0ae6265a", "metadata": {}, "outputs": [], "source": [ "# TODO: summarize fetched data amounts\n", "from collections import defaultdict\n", "val_durations_s = defaultdict(int)\n", "tr_durations_s = defaultdict(int)\n", "for m in sample_metas_val:\n", " val_durations_s[m[\"dataset\"]] += m[\"duration_s\"]\n", "for m in sample_metas_tr:\n", " tr_durations_s[m[\"dataset\"]] += m[\"duration_s\"]\n", "for k, v in tr_durations_s.items():\n", " print(f\"{v/60/60:,.1f} hours of {k} in train\") \n", "print()\n", "for k, v in val_durations_s.items():\n", " print(f\"{v/60/60:,.1f} hours of {k} in val\")" ] }, { "cell_type": "code", "execution_count": null, "id": "3c083a16", "metadata": {}, "outputs": [], "source": [ "assert(is_finished)\n", "write_jsonl(sample_metas_val, os.path.join(OUT_DATA_DIR, \"metas_val.jsonl\"))\n", "write_jsonl(sample_metas_tr, os.path.join(OUT_DATA_DIR, \"metas_tr.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "6cf7acd4", "metadata": {}, "outputs": [], "source": [ "# should be ~24h for ~10k hours\n", "# v1/youtube_music: 8.3 hours of data fetched in 156s and written in 9s as `val`, retained 100.0%\n", "# v1/youtube_music: 83.3 hours of data fetched in 782s and written in 63s as `train`, retained 100.0%" ] }, { "cell_type": "code", "execution_count": null, "id": "dd85de77", "metadata": { "scrolled": true }, "outputs": [], "source": [ "# test output\n", "m = random.choice(sample_metas_val)\n", "print(m[\"dataset\"])\n", "a = Audio.from_file(m[\"filepath\"], n_channels=2)\n", "a.play()\n", "# plt.plot(a.array_float[0]);" ] }, { "cell_type": "code", "execution_count": null, "id": "b0d95ab3", "metadata": {}, "outputs": [], "source": [ "# !du -hs /app/suno/data/audio_2ch_48khz_lg\n", "# # ~21T" ] }, { "cell_type": "code", "execution_count": null, "id": "72102a27", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "e8a46b8a", "metadata": {}, "source": [ "### Make DAC manifests" ] }, { "cell_type": "code", "execution_count": null, "id": "7732875b", "metadata": {}, "outputs": [], "source": [ "import os\n", "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"" ] }, { "cell_type": "code", "execution_count": null, "id": "854cd065", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "\n", "from suno_utils.utils.text import read_jsonl, read_json\n", "\n", "OUT_DATA_DIR = \"/app/suno/data/audio_2ch_48khz_lg\"" ] }, { "cell_type": "code", "execution_count": null, "id": "520dc8c3", "metadata": {}, "outputs": [], "source": [ "metas_tr = read_jsonl(os.path.join(OUT_DATA_DIR, \"metas_tr.jsonl\"))\n", "metas_val = read_jsonl(os.path.join(OUT_DATA_DIR, \"metas_val.jsonl\"))" ] }, { "cell_type": "code", "execution_count": null, "id": "edef2845", "metadata": { "scrolled": true }, "outputs": [], "source": [ "MUSIC_DATASETS = set([\n", " \"podcasts\",\n", " \"genius_hq\",\n", " \"youtube_music\",\n", " \"jamendo\",\n", " \"imslp\",\n", " \"pond5_music\",\n", " \"spot_genres\",\n", " \"tency\",\n", " \"shutter_music\",\n", " \"pond5_sfx\",\n", "])\n", "\n", "metas_tr = [m for m in metas_tr if m[\"dataset\"] in MUSIC_DATASETS]\n", "metas_val = [m for m in metas_val if m[\"dataset\"] in MUSIC_DATASETS]\n", "print(f\"{sum(m['duration_s'] for m in metas_tr)/60/60:,.0f} hours in train\")\n", "print(f\"{sum(m['duration_s'] for m in metas_val)/60/60:,.0f} hours in val\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5caa953d", "metadata": {}, "outputs": [], "source": [ "random.seed(7007)\n", "random.shuffle(metas_tr)\n", "random.shuffle(metas_val)" ] }, { "cell_type": "code", "execution_count": null, "id": "7e964f76", "metadata": {}, "outputs": [], "source": [ "df_tr = pd.DataFrame([{\"path\": m[\"filepath\"]} for m in metas_tr])\n", "df_val = pd.DataFrame([{\"path\": m[\"filepath\"]} for m in metas_val])" ] }, { "cell_type": "code", "execution_count": null, "id": "37d884dd", "metadata": {}, "outputs": [], "source": [ "df_tr.to_csv(os.path.join(OUT_DATA_DIR, \"music_tr.csv\"), index=False)\n", "df_val.to_csv(os.path.join(OUT_DATA_DIR, \"music_val.csv\"), index=False)" ] }, { "cell_type": "code", "execution_count": null, "id": "9087d847", "metadata": {}, "outputs": [], "source": [ "# # verify format\n", "# import csv\n", "# n = 0\n", "# with open(os.path.join(OUT_DATA_DIR, \"music_val.csv\"), \"r\") as f:\n", "# reader = csv.DictReader(f)\n", "# for x in reader:\n", "# print(x[\"path\"])\n", "# n+=1\n", "# if n == 5:\n", "# break" ] }, { "cell_type": "code", "execution_count": null, "id": "9934a968", "metadata": {}, "outputs": [], "source": [ "# # data breakdown\n", "\n", "# 30,491 hours in train\n", "\n", "# 2,624.5 hours of podcasts in train\n", "# 5,106.4 hours of genius_hq in train\n", "# 10,032.3 hours of youtube_music in train\n", "# 1,099.1 hours of jamendo in train\n", "# 1,045.7 hours of imslp in train\n", "# 2,550.8 hours of pond5_music in train\n", "# 6,446.1 hours of spot_genres in train\n", "# 502.3 hours of tency in train\n", "# 580.1 hours of shutter_music in train\n", "# 503.9 hours of pond5_sfx in train\n", "\n", "# 113 hours in val\n", "\n", "# 14.6 hours of podcasts in val\n", "# 14.1 hours of genius_hq in val\n", "# 14.6 hours of youtube_music in val\n", "# 13.5 hours of jamendo in val\n", "# 13.3 hours of imslp in val\n", "# 8.4 hours of pond5_music in val\n", "# 14.4 hours of spot_genres in val\n", "# 5.0 hours of tency in val\n", "# 14.4 hours of shutter_music in val\n", "# 1.1 hours of pond5_sfx in val\n", "\n", "# # general metas\n", "# /app/suno/data/audio_2ch_48khz_lg/metas_tr.jsonl\n", "# /app/suno/data/audio_2ch_48khz_lg/metas_val.jsonl\n", "\n", "# # codec train metas\n", "# /app/suno/data/audio_2ch_48khz_lg/music_tr.csv\n", "# /app/suno/data/audio_2ch_48khz_lg/music_val.csv" ] }, { "cell_type": "code", "execution_count": null, "id": "eedf6826", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1de5ac5c", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "658e72e8", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "id": "a49bc7e6", "metadata": {}, "source": [ "### misc" ] }, { "cell_type": "code", "execution_count": null, "id": "84b32343", "metadata": {}, "outputs": [], "source": [ "# prep shutterstock data\n", "new_metas = []\n", "for m in raw_shutter_metas:\n", " m_new = {\n", " \"id\": m[\"id\"],\n", " \"s3_filepath\": f\"s3://suno-data/datasets/harvest/shutterstock/audio/{m['id']}/{m['id']}.mp3\",\n", " \"duration_s\": m[\"nativeDuration\"],\n", " }\n", " stems = []\n", " n_stem = 0\n", " for mm in m[\"trackAssets\"][\"stems\"]:\n", " stems.append({\n", " \"id\": f\"{m['id']}_stem_{n_stem}\",\n", " \"s3_filepath\": f\"s3://suno-data/datasets/harvest/shutterstock/audio/{m['id']}/{m['id']}_{mm['assetClass']}.mp3\",\n", " \"duration_s\": m[\"nativeDuration\"],\n", " })\n", " n_stem += 1\n", " if len(stems) > 0:\n", " m_new[\"stems\"] = stems\n", " new_metas.append(m_new)" ] }, { "cell_type": "code", "execution_count": null, "id": "ffee3911", "metadata": {}, "outputs": [], "source": [ "!aws s3 ls s3://suno-data/datasets/harvest/tency/audio/42513/" ] }, { "cell_type": "code", "execution_count": null, "id": "e2ba8e31", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5f24144e", "metadata": {}, "outputs": [], "source": [ "!aws s3 cp s3://suno-data/datasets/harvest/tency/tency.jsonl tmp/" ] }, { "cell_type": "code", "execution_count": null, "id": "d3972361", "metadata": {}, "outputs": [], "source": [ "!tail -2 tmp/tency.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "48e1294e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "1d3642dd", "metadata": {}, "outputs": [], "source": [ "metas = read_from_s3(\"s3://suno-data/datasets/harvest/tency/tency.jsonl\", read_f=read_jsonl)" ] }, { "cell_type": "code", "execution_count": null, "id": "1cedf7e0", "metadata": {}, "outputs": [], "source": [ "metas[0]" ] }, { "cell_type": "code", "execution_count": null, "id": "5f6fb3d4", "metadata": {}, "outputs": [], "source": [ "!aws s3 ls s3://suno-data/datasets/harvest/tency/audio/61155/" ] }, { "cell_type": "code", "execution_count": null, "id": "4fd16bee", "metadata": {}, "outputs": [], "source": [ "from suno_utils.utils.s3 import list_s3_dir\n", "from joblib import Parallel, delayed\n", "\n", "id_list = [m[\"ID\"] for m in metas]" ] }, { "cell_type": "code", "execution_count": null, "id": "f3e70858", "metadata": {}, "outputs": [], "source": [ "def foo(_id):\n", " try:\n", " fd = f\"s3://suno-data/datasets/harvest/tency/audio/{_id}/\"\n", " return [os.path.join(fd, e) for e, _ in list_s3_dir(fd)]\n", " except:\n", " pass\n", " return None\n", "\n", "out = Parallel(n_jobs=16, backend=\"threading\")(\n", " delayed(foo)(_id) for _id in tqdm.tqdm(id_list)\n", ")" ] }, { "cell_type": "code", "execution_count": null, "id": "0c12611d", "metadata": {}, "outputs": [], "source": [ "metas_plus = []\n", "for _id, fps in zip(id_list, out):\n", " metas_plus.append({\n", " \"id\": _id,\n", " \"stems\": [\n", " {\n", " \"id\": f\"{_id}_stem_{n}\",\n", " \"s3_filepath\": fp,\n", " } \n", " for n, fp in enumerate(fps)\n", " ]\n", " })\n", "len(metas_plus)" ] }, { "cell_type": "code", "execution_count": null, "id": "566fa362", "metadata": {}, "outputs": [], "source": [ "metas_plus_flat = []\n", "for m in metas_plus:\n", " metas_plus_flat.extend(m[\"stems\"])" ] }, { "cell_type": "code", "execution_count": null, "id": "512da47c", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas_plus, \"tmp/metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "79192085", "metadata": {}, "outputs": [], "source": [ "!aws s3 cp tmp/metas.jsonl s3://suno-data/datasets/harvest/tency/tency_plus_fp.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "5a495dde", "metadata": {}, "outputs": [], "source": [ "write_jsonl(metas_plus_flat, \"tmp/metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "b96f0ff0", "metadata": {}, "outputs": [], "source": [ "!aws s3 cp tmp/metas.jsonl s3://suno-data/datasets/harvest/tency/tency_plus_fp_flat.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "29edc8af", "metadata": {}, "outputs": [], "source": [ "!head -3 tmp/metas.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "d4a00f03", "metadata": {}, "outputs": [], "source": [ "!tail -3 tmp/metas.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "b68b7c33", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "7565cd63", "metadata": {}, "outputs": [], "source": [ "# prep shutterstock data\n", "new_metas_ytm = []\n", "for m in metas_ytm:\n", " m_new = {\n", " \"id\": m[\"videoId\"],\n", " \"s3_filepath\": m[\"s3_filepath\"],\n", " \"duration_s\": m[\"duration_seconds\"],\n", " }\n", " new_metas_ytm.append(m_new)" ] }, { "cell_type": "code", "execution_count": null, "id": "96858817", "metadata": {}, "outputs": [], "source": [ "random.choice(new_metas_ytm)" ] }, { "cell_type": "code", "execution_count": null, "id": "24c8b088", "metadata": {}, "outputs": [], "source": [ "for m in metas_ytm:\n", " if m[\"videoId\"] == \"Y1KLTtUYCWU\":\n", " print(\"found\")\n", " break\n", "m" ] }, { "cell_type": "code", "execution_count": null, "id": "0c117d14", "metadata": {}, "outputs": [], "source": [ "Audio.from_s3('s3://suno-data/datasets/harvest/ytm_large/audio/Y1KLTtUYCWU.webm').play()" ] }, { "cell_type": "code", "execution_count": null, "id": "f7c8aed0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bf9c5f5a", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b7e27d3c", "metadata": {}, "outputs": [], "source": [ "write_jsonl(new_metas_ytm, \"tmp/metas.jsonl\")" ] }, { "cell_type": "code", "execution_count": null, "id": "61ed8e49", "metadata": {}, "outputs": [], "source": [ "!aws s3 cp tmp/metas.jsonl s3://suno-data/datasets/harvest/spotify/ytm_spotify_genres_50_simple.jsonl" ] }, { "cell_type": "code", "execution_count": null, "id": "41497430", "metadata": {}, "outputs": [], "source": [ "len(new_metas_ytm)" ] }, { "cell_type": "code", "execution_count": null, "id": "2275f7d2", "metadata": {}, "outputs": [], "source": [ "!aws s3 cp s3://suno-data/datasets/harvest/ytm_large/audio/GCNoRgIDWeY.webm ." ] }, { "cell_type": "code", "execution_count": null, "id": "0a0db833", "metadata": {}, "outputs": [], "source": [ "!ffprobe GCNoRgIDWeY.webm" ] }, { "cell_type": "code", "execution_count": null, "id": "92b723c5", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f5c3fbfc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "a5e9afef", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8854b0f1", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }