{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "import pandas as pd\n", "import numpy as np\n", "from suno_utils.utils.text import read_jsonl\n", "from suno_utils.utils.s3 import read_from_s3" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def analyze_audio(filepath):\n", " # check for metadata file\n", " metadata = None\n", " metadata_dict = None\n", " metadata_filepath = filepath.replace(\".mp3\", \"__metadata.npz\")\n", " if os.path.exists(metadata_filepath):\n", " try:\n", " #with open(metadata_filepath, \"r\") as f:\n", " # metadata = json.load(f)\n", " metadata = np.load(metadata_filepath, allow_pickle=True)\n", " # convert the metadata to a dictionary\n", " metadata_dict = {}\n", " for key, value in metadata.items():\n", " if key == \"diffusion\":\n", " # Convert numpy array back to dict if it was stored as such\n", " if isinstance(value, np.ndarray) and value.dtype == object:\n", " value = value.item()\n", " for k, v in value.items():\n", " metadata_dict[f\"diffusion_{k}\"] = v\n", " if key == \"hoot_cer\":\n", " if value == None:\n", " metadata_dict[key] = 0\n", " else:\n", " metadata_dict[key] = value\n", " else:\n", " metadata_dict[key] = value\n", "\n", " except Exception as e:\n", " print(f\"Error loading metadata: {e}\")\n", " metadata_dict = None\n", "\n", " return metadata_dict\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# load original metas to get the potential s3 ids\n", "work_items = read_from_s3(\n", " \"s3://suno-data/christian/sft/pos_interesting_clips_up_u_1_20241201_full.jsonl\",\n", " read_f=read_jsonl,\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# get ids\n", "original_ids = [f[\"id\"] for f in work_items]\n", "model_name = \"16n_25hz_v45_infill_shared_flow_resume_1_75m\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# copy from s3 to local\n", "aws s3 sync s3://suno-data/christian/outputs/v3-bootstrap-data-t0/ /app2/suno/data/christian/outputs/v3-bootstrap-data-t0/" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "base_s3_filepath = \"s3://suno-data/christian/outputs/v3-bootstrap-data-t2/\"\n", "local_base_filepath = \"/app2/suno/data/christian/outputs/v3-bootstrap-data-t2/\"\n", "\n", "# put metadata into a pandas dataframe\n", "metadata_list = []\n", "\n", "import glob\n", "# get all dirs in the local base filepath\n", "local_dirs = glob.glob(os.path.join(local_base_filepath, \"*\"))\n", "\n", "for local_dir in local_dirs:\n", " if os.path.exists(local_dir):\n", " clips = []\n", " # get all the metadata files in the directory\n", " metadata_files = [f for f in os.listdir(local_dir) if f.endswith(\"metadata.npz\")]\n", " for metadata_file in metadata_files:\n", " metadata = analyze_audio(os.path.join(local_dir, metadata_file))\n", " if \"filename\" not in metadata:\n", " continue\n", " clips.append(metadata)\n", " # only take two clips\n", " if len(clips) > 2:\n", " clips = clips[:2]\n", " if len(clips) < 2:\n", " continue\n", " for clip in clips:\n", " metadata_list.append(clip)\n", "\n", "metadata_df = pd.DataFrame(metadata_list)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metadata_df" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metadata_df.columns" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "for idx, row in metadata_df.iterrows():\n", " if row[\"hoot_cer\"] == None:\n", " print(row)\n" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "# create a comparision df by comparing the two clips\n", "# iterate over the pairs (even and odd)\n", "comparision_df_list = []\n", "for i in range(0, len(metadata_df), 2):\n", " even_clip = metadata_df.iloc[i]\n", " odd_clip = metadata_df.iloc[i+1]\n", " comparision_dict = {\n", " \"even_clip_steps\": even_clip[\"diffusion_steps\"],\n", " \"odd_clip_steps\": odd_clip[\"diffusion_steps\"],\n", " \"even_clip_shimmer\": even_clip[\"shimmer_score\"],\n", " \"odd_clip_shimmer\": odd_clip[\"shimmer_score\"],\n", " \"even_clip_ear_score\": even_clip[\"ear_score\"],\n", " \"odd_clip_ear_score\": odd_clip[\"ear_score\"],\n", " \"even_clip_hoot_cer\": even_clip[\"hoot_cer\"],\n", " \"odd_clip_hoot_cer\": odd_clip[\"hoot_cer\"],\n", " }\n", " comparision_df_list.append(comparision_dict)\n", "comparision_df = pd.DataFrame(comparision_df_list)\n" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Create boxplot using pandas but customize with matplotlib\n", "fig, axs = plt.subplots(figsize=(5, 12), nrows=5, sharex=False)\n", "\n", "# Get unique diffusion steps and create colormap\n", "unique_steps = sorted(metadata_df['diffusion_steps'].unique())\n", "cmap = plt.cm.viridis # You can change this to other colormaps like 'viridis', 'plasma', etc.\n", "colors = [cmap(0.1 + 0.7 * (i / (len(unique_steps) - 1))) for i in range(len(unique_steps))]\n", "\n", "# do this for ear_score and shimmer_score\n", "for i, column in enumerate([\"ear_score\", \"shimmer_score\", \"hoot_cer\", \"stereo_width\", \"lufs_db\"]):\n", " # Group and plot (ignore outliers)\n", " box = metadata_df.boxplot(\n", " column=column, \n", " by='diffusion_steps', \n", " grid=False, \n", " patch_artist=True, # Fills the boxes with color\n", " boxprops=dict(color='black'),\n", " medianprops=dict(color='black'),\n", " whiskerprops=dict(color='black'),\n", " capprops=dict(color='black'),\n", " flierprops=dict(marker='o', color='gray', alpha=0.5),\n", " ax=axs[i],\n", " showfliers=False\n", " )\n", "\n", " # Color each box according to diffusion steps\n", " # Get the box patches from the current axes\n", " box_patches = axs[i].findobj(plt.matplotlib.patches.PathPatch)\n", " for patch, step in zip(box_patches, unique_steps):\n", " step_index = unique_steps.index(step)\n", " patch.set_facecolor(colors[step_index])\n", " patch.set_alpha(0.75)\n", "\n", " axs[i].set_xlabel(\"Diffusion Steps\", fontsize=8)\n", " axs[i].set_ylabel(\"\", fontsize=12)\n", " axs[i].set_ylabel(column, fontsize=12)\n", " #\n", " # axs[i].set_title(f\"{column} Distribution by Diffusion Steps\", fontsize=14)\n", " #axs[i].suptitle(\"\") # Remove default 'Boxplot grouped by' title\n", " #axs[i].xticks(rotation=0)\n", " axs[i].grid(axis='y', linestyle='--', alpha=0.7)\n", "plt.tight_layout()\n", "plt.savefig(\"plots/boxplot_by_diffusion_steps.png\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ear score boxplot by sampler type\n", "fig, ax = plt.subplots(figsize=(5, 4))\n", "\n", "# Create boxplot for ear_score by diffusion_sampler_type\n", "metadata_df.boxplot(\n", " column='ear_score', \n", " by='diffusion_sampler_type', \n", " grid=False, \n", " patch_artist=True,\n", " boxprops=dict(facecolor='tab:blue', color='black', alpha=0.75),\n", " medianprops=dict(color='black'),\n", " whiskerprops=dict(color='black'),\n", " capprops=dict(color='black'),\n", " flierprops=dict(marker='o', color='gray', alpha=0.5),\n", " ax=ax\n", ")\n", "\n", "ax.set_xlabel(\"Sampler Type\", fontsize=12)\n", "ax.set_ylabel(\"Ear Score\", fontsize=12)\n", "ax.set_title(\"Ear Score Distribution by Sampler Type\", fontsize=14)\n", "ax.grid(axis='y', linestyle='--', alpha=0.7)\n", "plt.suptitle(\"\") # Remove default 'Boxplot grouped by' title\n", "plt.xticks(rotation=45)\n", "plt.tight_layout()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Memmap" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "# to create a memmap for sft we will select the higheset scoring upsample_id for each base_s3_id\n", "# we also need to grab the correct vae latents and semantic codes and text prompt\n", "from tqdm import tqdm\n", "from suno_utils.utils.text import write_jsonl\n", "import gc\n", "import sys\n", "import shutil\n", "\n", "# aws s3 sync s3://suno-data/christian/outputs/corrupt/genius_t6_sampled_10k/ /app/suno/data/diff_syn_dpo/genius_t6_sampled_10k_corrupted/npz\n", "\n", "SEMANTIC_RATE_HZ = 25\n", "CHUNK_SIZE_S = 30\n", "CHUNK_SIZE = int(CHUNK_SIZE_S * SEMANTIC_RATE_HZ)\n", "OUT_DATA_DIR = \"/app2/suno/data/christian/outputs/v3-bootstrap-data-t2/memmaps\"\n", "BASE_LOCAL_DIR = \"/app2/suno/data/christian/outputs/v3-bootstrap-data-t2/\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "\n", "metas = []\n", "\n", "# Get up to 1000 valid subdirectories\n", "local_dirs = [f for f in os.listdir(BASE_LOCAL_DIR) if os.path.isdir(os.path.join(BASE_LOCAL_DIR, f))]\n", "#local_dirs = local_dirs[:10000]\n", "\n", "# remove \"memmap\" from the list \n", "local_dirs = [f for f in local_dirs if \"memmap\" not in f]\n", "\n", "for local_dir in tqdm(local_dirs):\n", " dir_path = os.path.join(BASE_LOCAL_DIR, local_dir)\n", " \n", " # Find metadata files\n", " metadata_files = sorted([\n", " f for f in os.listdir(dir_path) \n", " if f.endswith(\"__metadata.npz\")\n", " ])\n", " \n", " # Require at least 2 files to form a pair\n", " if len(metadata_files) < 2:\n", " continue\n", " \n", " # Only use the first 2 (sorted so higher steps come last)\n", " metadata_files = metadata_files[:2]\n", "\n", " def extract_metadata(file_path):\n", " data = np.load(file_path, allow_pickle=True)\n", " md = {}\n", " for key, value in data.items():\n", " if key == \"diffusion\":\n", " if isinstance(value, np.ndarray) and value.dtype == object:\n", " value = value.item()\n", " for k, v in value.items():\n", " md[f\"diffusion_{k}\"] = v\n", " else:\n", " md[key] = value\n", " return md\n", "\n", " # Extract steps from filenames to identify positive/negative\n", " def get_step(file_name):\n", " parts = file_name.split(\"_steps_\")\n", " if len(parts) > 1:\n", " step_part = parts[1].split(\"__\")[0] # Safely handles __metadata\n", " return int(step_part)\n", " return 0\n", "\n", " steps_and_files = [(get_step(f), f) for f in metadata_files]\n", " steps_and_files.sort(key=lambda x: x[0]) # ascending: [negative, positive]\n", " \n", " # Get metadata for each file\n", " neg_step, neg_file = steps_and_files[0]\n", " pos_step, pos_file = steps_and_files[1]\n", " \n", " neg_path = os.path.join(dir_path, neg_file)\n", " pos_path = os.path.join(dir_path, pos_file)\n", "\n", " neg_meta = extract_metadata(neg_path)\n", " pos_meta = extract_metadata(pos_path)\n", "\n", " # Create entries\n", " neg_meta = {\n", " \"id\": local_dir,\n", " \"filename\": neg_meta[\"filename\"].item().replace(\".mp3\", \"\"),\n", " \"text\": neg_meta[\"text\"].item(),\n", " \"tags\": neg_meta[\"tags\"].item(),\n", " \"label\": \"negative\",\n", " \"step\": neg_step\n", " }\n", " pos_meta = {\n", " \"id\": local_dir,\n", " \"filename\": pos_meta[\"filename\"].item().replace(\".mp3\", \"\"),\n", " \"text\": pos_meta[\"text\"].item(),\n", " \"tags\": pos_meta[\"tags\"].item(),\n", " \"label\": \"positive\",\n", " \"step\": pos_step\n", " }\n", " # confirm that the text and tags are the same\n", " # if they are not the same continue\n", " if neg_meta[\"text\"] != pos_meta[\"text\"] or neg_meta[\"tags\"] != pos_meta[\"tags\"]:\n", " continue\n", "\n", " metas.append({\"negative\": neg_meta, \"positive\": pos_meta})\n", "\n", "print(f\"Collected {len(metas)} metadata entries.\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(metas[0])\n" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "# split the metas into tr and val\n", "metas_tr = metas[:int(len(metas)*0.9)]\n", "metas_val = metas[int(len(metas)*0.9):]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "if not os.path.exists(OUT_DATA_DIR):\n", " os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "else:\n", " #shutil.rmtree(OUT_DATA_DIR)\n", " os.makedirs(OUT_DATA_DIR, exist_ok=True)\n", "\n", "for dset_type in [\"tr\", \"val\"]:\n", "\n", " if dset_type == \"tr\":\n", " metas = metas_tr\n", " else:\n", " metas = metas_val\n", "\n", " new_metas = []\n", "\n", " out_mm_semantic_filepath = os.path.join(OUT_DATA_DIR, f\"data_semantic_{dset_type}.bin\")\n", " out_mm_vae_filepath = os.path.join(OUT_DATA_DIR, f\"data_vae_{dset_type}.bin\")\n", " out_metas_filepath = os.path.join(OUT_DATA_DIR, f\"metas_{dset_type}.jsonl\")\n", "\n", " n_offs_v = 0\n", " n_offs_s = 0\n", " to_write_len_v = 0\n", " to_write_len_s = 0\n", " total_hours = 0 # Counter for total hours of audio\n", "\n", " out_mm_semantic = np.memmap(\n", " out_mm_semantic_filepath, dtype=np.uint16, mode=\"w+\", shape=(1,)\n", " )\n", "\n", " out_mm_vae = np.memmap(\n", " out_mm_vae_filepath, dtype=np.float16, mode=\"w+\", shape=(1,)\n", " )\n", "\n", " # clear the metas file\n", " with open(out_metas_filepath, \"w\") as f:\n", " f.write(\"\")\n", "\n", " # Create a tqdm progress bar with hours counter\n", " pbar = tqdm(metas)\n", " pbar.set_description(\"Hours: 0.00\")\n", "\n", " for idx, (meta) in enumerate(pbar):\n", "\n", " pos_meta = meta[\"positive\"]\n", " neg_meta = meta[\"negative\"]\n", "\n", " example_id = pos_meta[\"id\"]\n", "\n", " # load semantic codes from disk\n", " semantic_codes_filepath = f\"{BASE_LOCAL_DIR}/{example_id}/{example_id}_semantic.npz\"\n", " pos_vae_latents_filepath = f\"{BASE_LOCAL_DIR}/{example_id}/{pos_meta['filename']}_upsampled_vae.npz\"\n", " neg_vae_latents_filepath = f\"{BASE_LOCAL_DIR}/{example_id}/{neg_meta['filename']}_upsampled_vae.npz\"\n", "\n", " try:\n", " semantic_data = np.load(semantic_codes_filepath)[\"semantic_codes\"]\n", " except Exception as e:\n", " print(f\"Error loading {semantic_codes_filepath}: {e}\")\n", " continue\n", "\n", " try:\n", " vae_data_pos = np.load(pos_vae_latents_filepath)[\"vae_latents\"].astype(np.float16)\n", " except Exception as e:\n", " print(f\"Error loading {pos_vae_latents_filepath}: {e}\")\n", " continue\n", "\n", " try:\n", " vae_data_neg = np.load(neg_vae_latents_filepath)[\"vae_latents\"].astype(np.float16)\n", " except Exception as e:\n", " print(f\"Error loading {neg_vae_latents_filepath}: {e}\")\n", " continue\n", "\n", " num_chunks = semantic_data.shape[0] // CHUNK_SIZE\n", "\n", " to_write_len_s = semantic_data[:750].size * num_chunks * 2\n", " to_write_len_v = vae_data_pos[:750, :].size * num_chunks * 2\n", " \n", " if to_write_len_s == 0:\n", " continue\n", "\n", " if to_write_len_v == 0:\n", " continue\n", " \n", " out_mm_semantic = np.memmap(\n", " out_mm_semantic_filepath,\n", " dtype=np.uint16,\n", " mode=\"r+\",\n", " shape=(n_offs_s + to_write_len_s,),\n", " )\n", "\n", " out_mm_vae = np.memmap(\n", " out_mm_vae_filepath,\n", " dtype=np.float16,\n", " mode=\"r+\",\n", " shape=(n_offs_v + to_write_len_v,),\n", " )\n", " # Add to total hours counter\n", " audio_duration_hours = (num_chunks * CHUNK_SIZE_S) / 3600\n", " total_hours += audio_duration_hours\n", " \n", " # Update progress bar description with current total hours\n", " pbar.set_description(f\"Hours: {total_hours:.2f}\")\n", "\n", " # we will write the positive and negative as interleaved chunks\n", " for i in range(num_chunks):\n", " for vae_data, steps in [(vae_data_neg, neg_meta[\"step\"]), (vae_data_pos, pos_meta[\"step\"])]:\n", " # create a new meta\n", " new_meta = {\n", " \"id\": pos_meta[\"id\"],\n", " \"start_s\": i*CHUNK_SIZE_S,\n", " \"end_s\": (i+1)*CHUNK_SIZE_S,\n", " \"original_duration_s\": semantic_data.shape[0] / SEMANTIC_RATE_HZ,\n", " \"n_vae_tokens\": CHUNK_SIZE,\n", " \"n_semantic_tokens\": CHUNK_SIZE,\n", " \"text\" : pos_meta[\"text\"],\n", " \"tags\" : pos_meta[\"tags\"],\n", " \"steps\" : steps\n", " }\n", " new_metas.append(new_meta)\n", "\n", " semantic_chunk = semantic_data[i*CHUNK_SIZE:(i+1)*CHUNK_SIZE]\n", "\n", " out_mm_semantic[n_offs_s : n_offs_s + semantic_chunk.size] = semantic_chunk.reshape(\n", " -1,\n", " )\n", " n_offs_s += semantic_chunk.size\n", "\n", " vae_chunk = vae_data[i*CHUNK_SIZE:(i+1)*CHUNK_SIZE, :]\n", " out_mm_vae[n_offs_v : n_offs_v + vae_chunk.size] = vae_chunk.reshape(\n", " -1,\n", " )\n", " n_offs_v += vae_chunk.size\n", "\n", " print(f\"Total hours of audio added: {total_hours:.4f} for {dset_type} set\")\n", " \n", " write_jsonl(\n", " new_metas,\n", " os.path.join(out_metas_filepath),\n", " do_append=True\n", " )\n", "\n", " out_mm_semantic.flush()\n", " out_mm_vae.flush()\n", " del out_mm_semantic, out_mm_vae, f\n", " gc.collect()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# test loading the memmaps \n", "\n", "VAE_DIM = 128\n", "VAE_N_MEMMAP_TOKENS = 750\n", "\n", "SEMANTIC_N_CODEBOOKS = 1\n", "SEMANTIC_N_MEMMAP_TOKENS = 750\n", "\n", "base_dir = \"/app2/suno/data/christian/outputs/v3-bootstrap-data-t2/memmaps\"\n", "\n", "metas = read_jsonl(f\"{base_dir}/metas_tr.jsonl\", progress=True)\n", "vae_memmap_filepath = f\"{base_dir}/data_vae_tr.bin\"\n", "semantic_memmap_filepath = f\"{base_dir}/data_semantic_tr.bin\"\n", "\n", "# load memmaps\n", "vae_memmap = np.memmap(vae_memmap_filepath, dtype=np.float16, mode=\"r\")\n", "semantic_memmap = np.memmap(semantic_memmap_filepath, dtype=np.uint16, mode=\"r\")\n", "\n", "# reshape memmaps\n", "vae_data = vae_memmap.reshape(-1, VAE_N_MEMMAP_TOKENS, VAE_DIM)\n", "semantic_data = semantic_memmap.reshape(-1, SEMANTIC_N_MEMMAP_TOKENS, SEMANTIC_N_CODEBOOKS)\n", "\n", "print(vae_data.shape, semantic_data.shape, len(metas))\n" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "# test some of the local data\n", "CODEC_FILEPATH = \"s3://suno-data/minz/models/dac_vae_tuned_25hz.pth\"\n", "\n", "from suno_utils.tasks.dac_vae_fixed_25hz import (\n", " preload_models as preload_codec_models,\n", " decode as codec_decode,\n", " encode as codec_encode,\n", " decode_stream_to_full_audio,\n", ")\n", "\n", "_ = preload_codec_models(CODEC_FILEPATH)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "idx = 138\n", "\n", "\n", "meta = metas[idx]\n", "#print(meta[\"tags\"], meta[\"text\"], meta[\"steps\"])\n", "print(metas[idx][\"steps\"], metas[idx+1][\"steps\"])\n", "vae_chunk_neg = vae_data[idx]\n", "vae_chunk_pos = vae_data[idx+1]\n", "semantic_chunk = semantic_data[idx]\n", "\n", "audio_neg = codec_decode(vae_chunk_neg).play()\n", "audio_pos = codec_decode(vae_chunk_pos).play()\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metas = pd.read_pickle(\n", " \"/home/tony/Data/Preference/up_v2_d4/interesting_clips_ahi_d4_20250623.pkl\"\n", ")\n", "print(len(metas))\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metas.iloc[2].values" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# convert this into metas jsonl \n", "\n", "# First check what columns are available\n", "print(\"Available columns:\", metas.columns.tolist())\n", "print(\"Sample row:\", metas.iloc[0] if len(metas) > 0 else \"No data\")\n", "\n", "new_metas = []\n", "\n", "for idx, row in metas.iterrows():\n", " # if idx is odd then it is negative, even is positive\n", " # only take the positive, skip the negative\n", " if idx % 2 == 0:\n", " continue\n", " metadata = row[\"metadata\"]\n", " upsample_id = metadata.get(\"upsample_clip_id\", None)\n", " if upsample_id is None:\n", " continue\n", " meta_dict = {\n", " \"id\": metadata[\"upsample_clip_id\"], # this is the parent_id\n", " \"upsample_id\": row[\"id\"], # this is the upsample_id\n", " \"text\": row[\"prompt_text\"],\n", " \"tags\": metadata[\"tags\"]\n", " }\n", " \n", "\n", " new_metas.append(meta_dict)\n", "\n", "\n", "write_jsonl(\n", " new_metas,\n", " \"/home/christian/code/christian/metadata/sft/auk_clips_up_v2_d4_ahi_d4_20250623_pos.jsonl\"\n", ")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metas.iloc[0][\"metadata\"]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "new_metas[6]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(len(new_metas))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "suno_diff", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.9" } }, "nbformat": 4, "nbformat_minor": 2 }