#!/bin/bash
#SBATCH --job-name=process_podcasts
#SBATCH --nodes=8
#SBATCH --gres=gpu:8
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=10

# other useful slurm commands
# --exclude=h100-ord01-03-[]
# --nodelist=h100-ord01-03-[]

# cluster specific defaults
export OMP_NUM_THREADS=1

# explicit cache dirs to not get user conflicts
export TRITON_CACHE_DIR=/mnt/localdisk/.triton_cache_$USER
export VERBOSE=0

# using bare python rather than torchrun prevents hanging on nightly
# -K1 means slurm job will crash if run crashes rather than just hang
# -u means python will stream stoud as it comes rather than buffer
echo "Starting podcast processing script..."
srun /home/victor/anaconda3/envs/dev/bin/python -u process_podcasts_with_hoot.py