import os import re import json import boto3 import argparse import sqlalchemy import pandas as pd from tqdm import tqdm from sqlalchemy import create_engine DEFAULT_N_CORES = 10 DEFAULT_CHUNKSIZE = 1_000 S3_BUCKET_PATH_RE = r"s3\:\/\/(.+?)\/" def get_filename(filepath, keep_ext=True): if "http" in filepath: clean_filepath = filepath.split("?")[0] else: clean_filepath = filepath filename = clean_filepath.split("/")[-1] if "." not in filename: raise ValueError("filename does not seem to contain a period.") m = re.search(r"(.+)\.([^\.]+)$", filename) if not m: raise ValueError(f"filename could not be parsed for `{filepath}`") filename = m.group(1) file_ext = m.group(2).lower() if len(file_ext) > 10: raise ValueError(f"file extension suspiciously long for `{filepath}`") if keep_ext: filename = filename + "." + file_ext return filename def get_file_ext(filepath): filename = get_filename(filepath, keep_ext=True) file_ext = filename.split(".")[-1] return file_ext def _parse_s3_filepath(s3_filepath: str): bucket_name = re.search(S3_BUCKET_PATH_RE, s3_filepath).group(1) rel_s3_filepath = re.sub(S3_BUCKET_PATH_RE, "", s3_filepath) return bucket_name, rel_s3_filepath def _get_client(client_config=None): if client_config is not None: client = boto3.client( "s3", endpoint_url=client_config["endpoint_url"], aws_access_key_id=client_config["aws_access_key_id"], aws_secret_access_key=client_config["aws_secret_access_key"], region_name=client_config["region_name"], ) else: client = boto3.client("s3") return client def _verify_s3_filepath(filepath): if re.search(S3_BUCKET_PATH_RE, filepath) is None: raise ValueError("not a valid s3 filepath") if get_file_ext(filepath) is None or len(get_file_ext(filepath)) == 0: raise ValueError("not a valid file extension") def download_from_s3(s3_filepath: str, local_filepath: str): bucket_name, from_rel_s3_filepath = _parse_s3_filepath(s3_filepath) client = _get_client() client.download_file(bucket_name, from_rel_s3_filepath, local_filepath) def grab(json_filepath: str, output_dir: str, num: int = 10): with open(json_filepath, "r") as fp: trending = json.load(fp) # construct s3 filepath s3_basepath = "s3://suno-data-uploads/studio/uploads/" for row_idx, row in enumerate(tqdm(trending)): s3_filepath = os.path.join(s3_basepath, f"""{(row["clip_id"])}.mp3""") print(s3_filepath) output_filepath = os.path.join(output_dir, f"""{row["clip_id"]}.mp3""") try: _verify_s3_filepath(s3_filepath) download_from_s3(s3_filepath, output_filepath) except Exception as e: print(e) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "json_filepath", help="Path to JSON file containing trending songs." ) parser.add_argument( "--output_dir", help="Path to directory to store downloaded audio files." ) parser.add_argument( "--num", help="Number of audio files to pull from database.", default=10 ) args = parser.parse_args() os.makedirs(args.output_dir) grab(args.json_filepath, args.output_dir)