"""MediaConvert testing integration""" import logging import os import subprocess import sys import tempfile import time from pathlib import Path from typing import Any, Dict, Optional from pipeline.constants import ( AWS_REGION, DEST_BUCKET, MEDIACONVERT_ENDPOINT, MEDIACONVERT_ROLE, SOURCE_BUCKET, SUNO_DATA_UPLOADS_BUCKET, ) from pipeline.s3_operations import S3Client logger = logging.getLogger(__name__) class MediaConvertTester: """Client for testing videos with AWS MediaConvert""" def __init__( self, s3_client: S3Client, glockenspiel_repo: str = None, ): self.s3_client = s3_client self.glockenspiel_repo = glockenspiel_repo or os.getenv( "GLOCKENSPIEL_REPO", "/Users/mark/dev/glockenspiel" ) def test_video( self, hook_id: str, input_s3_key: str, skip_copy: bool = False, ) -> Dict[str, Any]: """Run the test_mediaconvert.py test on the output file Args: input_s3_key: S3 key of the input video output_prefix: S3 prefix for output files Returns: Dictionary with MediaConvert job results """ logger.info("Running MediaConvert test...") source_s3_key = f"tests/hook-transcode-test/transcode-test-{hook_id}.mp4" if skip_copy: logger.info(f"Skipping copy of {input_s3_key} to {source_s3_key}") else: # Copy from the bucket where the video actually exists to SOURCE_BUCKET logger.info( f"Copying {input_s3_key} from {SUNO_DATA_UPLOADS_BUCKET} to {SOURCE_BUCKET}/{source_s3_key}" ) self.s3_client.copy_file( s3_bucket=SUNO_DATA_UPLOADS_BUCKET, # Source is in SUNO_DATA_UPLOADS_BUCKET s3_key=input_s3_key, s3_bucket_dest=SOURCE_BUCKET, # Copy TO the SOURCE_BUCKET for MediaConvert s3_key_dest=source_s3_key, ) # Import and run the test sys.path.insert( 0, os.path.join(self.glockenspiel_repo, "suno-cdk/python_tests") ) try: # Mock datadog before importing class MockDatadogLambda: @staticmethod def lambda_metric(name, value, tags=None): logger.debug(f"[Mock Datadog] {name}: {value}") sys.modules["datadog_lambda"] = MockDatadogLambda() sys.modules["datadog_lambda.metric"] = MockDatadogLambda # Set environment variables os.environ["SOURCE_BUCKET"] = SOURCE_BUCKET os.environ["DESTINATION_BUCKET"] = DEST_BUCKET os.environ["AWS_DEFAULT_REGION"] = AWS_REGION os.environ["MEDIACONVERT_ENDPOINT"] = MEDIACONVERT_ENDPOINT os.environ["MEDIACONVERT_ROLE"] = MEDIACONVERT_ROLE # Add lambda handler to path lambda_path = os.path.join( self.glockenspiel_repo, "suno-cdk/lambda/media-convert-handler" ) sys.path.insert(0, lambda_path) try: from media_convert_handler import ( create_hls_job_params, get_mediaconvert_client, ) except ImportError as e: logger.warning(f"Could not import MediaConvert handler: {e}") logger.warning("MediaConvert test skipped - handler not available") return { "status": "SKIPPED", "error": "MediaConvert handler not available", } # Create MediaConvert job mediaconvert_client = get_mediaconvert_client() # Create job parameters # The destination_path should be in the format: s3://bucket/prefix/ # Include hook_id as a folder for better organization # Note: MediaConvert needs trailing slash to treat it as a directory output_prefix = f"tests/hook-transcode-tests/{hook_id}/" destination_path = f"s3://{DEST_BUCKET}/{output_prefix}" logger.info(f"MediaConvert destination path: {destination_path}") job_params = create_hls_job_params( source_bucket=SOURCE_BUCKET, source_key=source_s3_key, destination_path=destination_path, environment="staging", skip_callback=True, # Skip callback for testing ) # Add custom metadata to identify this as a transcode test job # Merge with any existing metadata instead of replacing if "UserMetadata" not in job_params: job_params["UserMetadata"] = {} job_params["UserMetadata"].update( { "Type": "transcode-test", "HookId": hook_id, "Source": "hook-transcode-test-pipeline", } ) # Add tags for easier filtering in AWS console job_params["Tags"] = { "Type": "transcode-test", "Environment": "test", "Pipeline": "hook-transcode-test", } # Submit the job logger.info( f"Submitting MediaConvert transcode-test job for hook {hook_id}..." ) response = mediaconvert_client.create_job(**job_params) job_id = response["Job"]["Id"] logger.info(f"MediaConvert transcode-test job created: {job_id}") logger.info(f"Job metadata: Type=transcode-test, HookId={hook_id}") # Wait for job completion logger.info("Waiting for MediaConvert job...") while True: job_status = mediaconvert_client.get_job(Id=job_id) status = job_status["Job"]["Status"] if status == "COMPLETE": logger.info("MediaConvert job complete") break elif status in ["ERROR", "CANCELED"]: logger.error(f"MediaConvert job failed: {status}") raise Exception(f"MediaConvert job failed: {status}") time.sleep(5) return { "job_id": job_id, "status": "COMPLETE", "output_prefix": output_prefix.rstrip( "/" ), # Remove trailing slash for consistency } except Exception as e: logger.error(f"Error running MediaConvert test: {e}") return {"status": "ERROR", "error": str(e)} def download_and_concat_720p( self, bucket: str, prefix: str, output_dir: Path ) -> Optional[str]: """Download and concatenate 720p HLS segments to MP4 Args: bucket: S3 bucket containing HLS files prefix: S3 prefix where HLS files are located output_dir: Local directory to save concatenated MP4 Returns: Path to concatenated MP4 file or None if failed """ try: # Create directory for HLS files in the output directory hls_dir = output_dir / "hls_fragments" hls_dir.mkdir(exist_ok=True) temp_path = hls_dir # Download only 720p HLS files from S3 logger.info(f"Looking for 720p HLS files in s3://{bucket}/{prefix}") # Ensure prefix doesn't have double slashes # Remove trailing slash if present since we already included it clean_prefix = prefix.rstrip("/") # List all objects in the prefix response = self.s3_client.client.list_objects_v2( Bucket=bucket, Prefix=clean_prefix ) logger.debug( f"ListObjects response: {response.get('KeyCount', 0)} keys found" ) if "Contents" not in response: logger.warning("No HLS files found in S3") return None # Find and download only 720p-related files playlist_720p = None downloaded_files = [] for obj in response["Contents"]: key = obj["Key"] filename = Path(key).name # Download 720p playlist and its segments if "_720p" in filename or "720p" in filename: local_file = temp_path / filename logger.debug(f"Downloading 720p file: {filename}") self.s3_client.client.download_file( bucket, key, str(local_file) ) downloaded_files.append(filename) # Check if this is the 720p playlist if filename.endswith("_720p.m3u8"): playlist_720p = local_file if not playlist_720p: logger.warning("No 720p playlist found") return None logger.info(f"Downloaded {len(downloaded_files)} 720p files") logger.info(f"Found 720p playlist: {playlist_720p.name}") # Output MP4 file output_mp4 = output_dir / "3_mediaconvert_720p.mp4" # Use ffmpeg to concatenate - the TS->MP4 remux will create B-frame offset # We'll measure and fix it in a second pass temp_mp4 = output_dir / "3_mediaconvert_720p_temp.mp4" cmd = [ "ffmpeg", "-i", str(playlist_720p), "-c", "copy", "-movflags", "+faststart", "-y", # Overwrite output "-loglevel", "error", # Reduce verbosity str(temp_mp4), ] logger.info(f"Concatenating 720p segments (temp)...") subprocess.run(cmd, capture_output=True, text=True, check=True) # Probe the temp file to get actual video start time after TS->MP4 remux probe_cmd = [ "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=start_time", "-of", "default=noprint_wrappers=1:nokey=1", str(temp_mp4), ] probe_result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True) video_start_time = float(probe_result.stdout.strip() or "0") logger.info(f"Video start time after TS->MP4 remux: {video_start_time}s") # Second pass: fix timestamps using measured offset if abs(video_start_time) > 0.001: # Only fix if offset is significant cmd = [ "ffmpeg", "-i", str(temp_mp4), "-c", "copy", "-output_ts_offset", str(-video_start_time), # Shift both streams to align at 0 "-movflags", "+faststart", "-y", "-loglevel", "error", str(output_mp4), ] logger.info(f"Fixing timestamps with offset: -{video_start_time}s") subprocess.run(cmd, capture_output=True, text=True, check=True) temp_mp4.unlink() # Clean up temp file else: temp_mp4.rename(output_mp4) # No fix needed, just rename logger.info("No timestamp fix needed") # Get file size size_mb = output_mp4.stat().st_size / (1024 * 1024) logger.info(f"✅ Created 720p MP4: {size_mb:.2f} MB") logger.info(f"HLS fragments saved to: {hls_dir}") return str(output_mp4) except subprocess.CalledProcessError as e: logger.error(f"Failed to concatenate 720p: {e}") if e.stderr: logger.error(f"FFmpeg error: {e.stderr}") return None except Exception as e: logger.error(f"Error downloading/concatenating 720p: {e}") return None