#include "ffmpegbuffer.h" #include "concurrent/appendonlybuffer.h" #include "util.h" #include #include #include #include #include #include #include #include #include #include #include #include #include extern "C" { #include #include #include #include #include #include #include } using encoded_data_vec_t = const std::vector; struct NodeJSDataSource : public FfmpegDataSource { size_t _size; emscripten::val _readCallback, _closeCallback; NodeJSDataSource(size_t size, emscripten::val _readCallback, emscripten::val _closeCallback) : _size(size), _readCallback(_readCallback), _closeCallback(_closeCallback) {} ~NodeJSDataSource() { _closeCallback(); } size_t size() override { return _size; } size_t read(size_t offset, size_t count, uint8_t *out) override { if (offset >= _size) { return 0; } size_t bytesToRead = std::min(count, _size - offset); auto bufJs = emscripten::val::global("Uint8Array") .new_(emscripten::val::module_property( "GROWABLE_HEAP_U8")()["buffer"], reinterpret_cast(out), bytesToRead); _readCallback(bufJs, offset); return bytesToRead; } bool isCompleted() const override { return true; } }; struct VectorDataSource : public FfmpegDataSource { std::vector data; VectorDataSource() = default; VectorDataSource(const std::vector &vec) : data(vec) {} VectorDataSource(std::vector &&vec) : data(std::move(vec)) {} template VectorDataSource(InputIterator begin, InputIterator end) : data(begin, end) {} VectorDataSource(std::initializer_list ilist) : data(ilist) {} size_t size() override { return data.size(); } size_t read(size_t offset, size_t count, uint8_t *out) override { if (offset >= data.size()) { return 0; } size_t bytesToRead = std::min(count, data.size() - offset); memcpy(out, data.data() + offset, bytesToRead); return bytesToRead; } bool isCompleted() const override { return true; } }; struct AOBDataSource : public FfmpegDataSource { AppendOnlyBuffer data; int32_t completed; int32_t bitrate; // bitrate in bits per second AOBDataSource() : completed(0), bitrate(128000) {} size_t size() override { return data.size(); } size_t read(size_t offset, size_t count, uint8_t *out) override { return data.read(offset, count, out); } bool isCompleted() const override { return emscripten_atomic_load_u32(&completed) == 1; } void setBitrate(int bitrateValue) { emscripten_atomic_store_u32(&bitrate, bitrateValue); } int getBitrate() const { return emscripten_atomic_load_u32(&bitrate); } float getDecodedLength() const { int currentBitrate = getBitrate(); if (currentBitrate > 0) { size_t availableBytes = data.size(); float estimatedDuration = (float)availableBytes / ((float)currentBitrate / 8.0f); return estimatedDuration; } return 0.0f; } // called from JS void write(emscripten::val value) { // value: Uint8Array size_t size = value["length"].as(); std::vector buf(size); auto bufJs = emscripten::val::global("Uint8Array") .new_(emscripten::val::module_property( "GROWABLE_HEAP_U8")()["buffer"], reinterpret_cast(buf.data()), size); bufJs.call("set", value); data.append(size, buf.data()); } void setCompleted() { emscripten_atomic_store_u32(&completed, 1); } }; struct FfmpegLibraryEntry { std::weak_ptr data; std::list> waiters; }; class FfmpegLibrary { private: void clearDeadEntries() { for (auto it = tokenEntries.begin(); it != tokenEntries.end();) { if (it->second.data.lock() == nullptr && it->second.waiters.empty()) { // Remove URL mappings pointing to this token for (auto it2 = urlToToken.begin(); it2 != urlToToken.end();) { if (it2->second == it->first) { it2 = urlToToken.erase(it2); } else { ++it2; } } // Remove UUID mappings pointing to this token for (auto it2 = uuidToToken.begin(); it2 != uuidToToken.end();) { if (it2->second == it->first) { it2 = uuidToToken.erase(it2); } else { ++it2; } } it = tokenEntries.erase(it); } else { ++it; } } } // token -> data std::unordered_map tokenEntries; // url -> token std::unordered_map urlToToken; // uuid -> token std::unordered_map uuidToToken; // Next available token int nextToken; emscripten_lock_t lock; public: FfmpegLibrary() : nextToken(1) { emscripten_lock_init(&lock); } int getDataOrWait(const std::string &url, const std::optional &uuid, const std::shared_ptr &buffer) { int token = 0; std::shared_ptr maybeData; { emscripten_lock_raii guard(lock); // Check if URL already has a token auto urlIt = urlToToken.find(url); if (urlIt != urlToToken.end()) { token = urlIt->second; } else { // Create new token and mappings token = nextToken++; urlToToken[url] = token; if (uuid.has_value()) { uuidToToken[uuid.value()] = token; } } // Get or create entry for this token auto &entry = tokenEntries[token]; maybeData = entry.data.lock(); if (!maybeData) { entry.data.reset(); entry.waiters.push_back(buffer); clearDeadEntries(); return token; // Return token, data not ready yet } } // Data is ready, set it without holding the lock buffer->setData(maybeData); buffer->resolvePromise(buffer); return token; } bool getTokenDataOrWait(int token, const std::shared_ptr &buffer) { std::shared_ptr maybeData; { emscripten_lock_raii guard(lock); auto it = tokenEntries.find(token); if (it == tokenEntries.end()) { // Token doesn't exist return false; } auto &entry = it->second; maybeData = entry.data.lock(); if (!maybeData) { entry.data.reset(); entry.waiters.push_back(buffer); clearDeadEntries(); return true; } } // Data is ready, set it without holding the lock buffer->setData(maybeData); buffer->resolvePromise(buffer); return true; } bool getUuidDataOrWait(const std::string &uuid, const std::shared_ptr &buffer) { int token = 0; { emscripten_lock_raii guard(lock); auto uuidIt = uuidToToken.find(uuid); if (uuidIt == uuidToToken.end()) { // UUID not found, cannot trigger download return false; } token = uuidIt->second; } // Set the token on the buffer buffer->token = token; // Call getTokenDataOrWait without holding the lock return getTokenDataOrWait(token, buffer); } void setData(const std::string &url, const std::shared_ptr &data) { std::list> waitersToNotify; { emscripten_lock_raii guard(lock); auto urlIt = urlToToken.find(url); if (urlIt == urlToToken.end()) { // URL not found, this shouldn't happen in normal flow return; } int token = urlIt->second; auto &entry = tokenEntries[token]; entry.data = data; // Move waiters out of the critical section waitersToNotify = std::move(entry.waiters); entry.waiters = std::list>(); clearDeadEntries(); } // Notify waiters without holding the lock (setData is heavy) for (auto &waiter : waitersToNotify) { auto lockedWaiter = waiter.lock(); if (lockedWaiter) { lockedWaiter->setData(data); lockedWaiter->resolvePromise(lockedWaiter); } } } // Register data with a UUID immediately (for byte arrays and other direct // sources) int registerDataWithUuid(const std::string &uuid, const std::shared_ptr &data) { int token = 0; std::list> waitersToNotify; { emscripten_lock_raii guard(lock); // Check if UUID already has a token auto uuidIt = uuidToToken.find(uuid); if (uuidIt != uuidToToken.end()) { token = uuidIt->second; auto &entry = tokenEntries[token]; entry.data = data; // Move waiters out of the critical section waitersToNotify = std::move(entry.waiters); entry.waiters = std::list>(); } else { // Create new token and mappings token = nextToken++; uuidToToken[uuid] = token; tokenEntries[token] = {data, {}}; } clearDeadEntries(); } // Notify waiters without holding the lock (setData is heavy) for (auto &waiter : waitersToNotify) { auto lockedWaiter = waiter.lock(); if (lockedWaiter) { lockedWaiter->setData(data); lockedWaiter->resolvePromise(lockedWaiter); } } return token; } // Add an additional UUID alias for an existing UUID // Useful when transitioning from uploadId to clipId bool addUuidAlias(const std::string &existingUuid, const std::string &newUuid) { emscripten_lock_raii guard(lock); auto existingIt = uuidToToken.find(existingUuid); if (existingIt == uuidToToken.end()) { // Existing UUID not found return false; } int token = existingIt->second; // Map the new UUID to the same token uuidToToken[newUuid] = token; return true; } }; static FfmpegLibrary ffmpegLibrary; void downloadSucceeded(emscripten_fetch_t *fetch) { printf("Finished downloading %llu bytes from URL %s.\n", fetch->numBytes, fetch->url); ffmpegLibrary.setData(fetch->url, std::make_shared( fetch->data, fetch->data + fetch->numBytes)); emscripten_fetch_close(fetch); } void downloadFailed(emscripten_fetch_t *fetch) { printf("Downloading %s failed, HTTP failure status code: %d.\n", fetch->url, fetch->status); emscripten_fetch_close(fetch); } inline static void log_ffmpeg_error(int res, const std::string &step) { char errbuf[256]; memset(errbuf, 0, sizeof(errbuf)); av_strerror(res, errbuf, sizeof(errbuf)); std::string msg = "ffmpeg: Failed to " + step + ": " + errbuf; emscripten_console_log(msg.c_str()); } constexpr size_t io_buffer_size = 4096; struct FfmpegState { // ffmpeg context AVIOContext *io_context; AVFormatContext *format_context; int audio_stream_index; AVCodec *decoder; AVCodecContext *codec_context; AVPacket *packet; AVFrame *frame; SwrContext *resampler; // encoded data in memory std::shared_ptr data; size_t data_pos; // probed values int sample_rate; int channel_count; int frame_count; int zero_pts_frame_index; bool initialized; bool is_streaming; bool read_frame_err; // partially-consumed frame state int partially_consumed_offset; float *resampled_data; bool have_packet; FfmpegState(std::shared_ptr data) : io_context(nullptr), format_context(nullptr), audio_stream_index(0), decoder(nullptr), codec_context(nullptr), packet(nullptr), frame(nullptr), resampler(nullptr), data(std::move(data)), data_pos(0), sample_rate(48000), channel_count(2), frame_count(0), partially_consumed_offset(-1), zero_pts_frame_index(0), resampled_data(nullptr), have_packet(false), initialized(false), is_streaming(dynamic_cast(this->data.get()) != nullptr), read_frame_err(false) { // configure io context uint8_t *initial_io_buffer = reinterpret_cast(av_malloc(io_buffer_size)); if (initial_io_buffer == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate io buffer"); return; } io_context = avio_alloc_context(initial_io_buffer, io_buffer_size, 0, this, read_packet, nullptr, seek_packet); if (io_context == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate io context"); return; } // configure format context format_context = avformat_alloc_context(); if (format_context == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate format context"); return; } format_context->pb = io_context; int res = avformat_open_input(&format_context, nullptr, nullptr, nullptr); if (res < 0) { if (!(res == AVERROR(EAGAIN) && is_streaming)) { log_ffmpeg_error(res, "open input"); } return; } // find stream res = avformat_find_stream_info(format_context, nullptr); if (res < 0) { if (!(res == AVERROR(EAGAIN) && is_streaming)) { log_ffmpeg_error(res, "find stream info"); } return; } audio_stream_index = av_find_best_stream(format_context, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0); if (audio_stream_index < 0) { log_ffmpeg_error(audio_stream_index, "find best stream"); return; } if (format_context->streams[audio_stream_index]->codecpar->codec_type != AVMEDIA_TYPE_AUDIO) { emscripten_console_log("ffmpeg: Found non-audio stream"); return; } auto audio_stream = format_context->streams[audio_stream_index]; // find decoder auto decoder = avcodec_find_decoder(audio_stream->codecpar->codec_id); if (decoder == nullptr) { std::string codec_id_str = std::to_string(audio_stream->codecpar->codec_id); const AVCodecDescriptor *desc = avcodec_descriptor_get(audio_stream->codecpar->codec_id); if (desc) { codec_id_str = std::string(desc->name); } emscripten_console_log( ("ffmpeg: Failed to find decoder for codec id: " + codec_id_str) .c_str()); return; } // configure codec context codec_context = avcodec_alloc_context3(decoder); if (codec_context == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate codec context"); return; } res = avcodec_parameters_to_context(codec_context, audio_stream->codecpar); if (res < 0) { log_ffmpeg_error(res, "copy codec parameters to context"); return; } codec_context->pkt_timebase = audio_stream->time_base; res = avcodec_open2(codec_context, decoder, nullptr); if (res < 0) { log_ffmpeg_error(res, "open codec"); return; } // probe stream info sample_rate = codec_context->sample_rate; channel_count = codec_context->ch_layout.nb_channels; frame_count = 0; if (is_streaming) { const auto dataAOB = dynamic_cast(this->data.get()); if (dataAOB) { int bitrate = 0; if (codec_context->bit_rate > 0) { bitrate = codec_context->bit_rate; } else if (audio_stream->codecpar->bit_rate > 0) { bitrate = audio_stream->codecpar->bit_rate; } else if (format_context->bit_rate > 0) { bitrate = format_context->bit_rate; } else { bitrate = 128000; } dataAOB->setBitrate(bitrate); } } if (!(channel_count == 1 || channel_count == 2)) { emscripten_console_log(("ffmpeg: Expected 1 or 2 channels, got " + std::to_string(channel_count)) .c_str()); return; } if (audio_stream->duration != AV_NOPTS_VALUE) { frame_count = av_rescale_q(audio_stream->duration, audio_stream->time_base, {1, sample_rate}); } else if (format_context->duration != AV_NOPTS_VALUE) { frame_count = av_rescale_q(format_context->duration, AV_TIME_BASE_Q, {1, sample_rate}); } if (frame_count == 0 && audio_stream->nb_frames > 0) { frame_count = audio_stream->nb_frames; } if (frame_count == 0) { if (is_streaming) { frame_count = INT_MAX; } else { emscripten_console_log("ffmpeg: warning: Failed to determine frame " "count, things will probably break"); } } // alloc packet and frame packet = av_packet_alloc(); if (packet == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate packet"); return; } frame = av_frame_alloc(); if (frame == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate frame"); return; } // configure resampler AVChannelLayout input_ch_layout = {}, output_ch_layout = {}; if (codec_context->ch_layout.order != AV_CHANNEL_ORDER_UNSPEC) { res = av_channel_layout_copy(&input_ch_layout, &codec_context->ch_layout); if (res < 0) { log_ffmpeg_error(res, "copy input channel layout"); return; } } else { av_channel_layout_default(&input_ch_layout, channel_count); } av_channel_layout_default(&output_ch_layout, channel_count); // Use the newer swr_alloc_set_opts2 function resampler = nullptr; res = swr_alloc_set_opts2( &resampler, &output_ch_layout, // output channel layout AV_SAMPLE_FMT_FLT, // output sample format (32-bit float) codec_context->sample_rate, // output sample rate (same as input) &input_ch_layout, // input channel layout codec_context->sample_fmt, // input sample format codec_context->sample_rate, // input sample rate 0, nullptr); if (res < 0) { log_ffmpeg_error(res, "configure resampler"); return; } if (resampler == nullptr) { emscripten_console_log("ffmpeg: Failed to allocate resampler"); return; } res = swr_init(resampler); if (res < 0) { log_ffmpeg_error(res, "init resampler"); return; } // Clean up temporary channel layouts av_channel_layout_uninit(&input_ch_layout); av_channel_layout_uninit(&output_ch_layout); // Decode 1 frame to determine pts offset if (!nextFrame()) { return; } zero_pts_frame_index = av_rescale_q(frame->pts, codec_context->time_base, {1, sample_rate}); partially_consumed_offset = 0; resampleCurrentFrame(); initialized = true; } bool nextFrame() { for (;;) { if (!have_packet) { int read_result = av_read_frame(format_context, packet); if (read_result < 0) { if (is_streaming && read_result == AVERROR(EAGAIN)) { // No more data available yet in streaming mode return false; } // this kind of error is not recoverable; we must reinitialize the // decoder read_frame_err = true; return false; } if (packet->stream_index != audio_stream_index) { av_packet_unref(packet); continue; } if (avcodec_send_packet(codec_context, packet) < 0) { av_packet_unref(packet); continue; } have_packet = true; } int receive_res = avcodec_receive_frame(codec_context, frame); if (receive_res >= 0) { return true; } av_packet_unref(packet); have_packet = false; if (receive_res == AVERROR(EAGAIN)) { continue; } if (receive_res == AVERROR_EOF) { return false; } emscripten_console_log(("receive frame failed with error code " + std::to_string(receive_res)) .c_str()); return false; } } ~FfmpegState() { swr_free(&resampler); if (partially_consumed_offset >= 0) { av_freep(&resampled_data); partially_consumed_offset = -1; } if (have_packet) { av_packet_unref(packet); have_packet = false; } av_frame_free(&frame); av_packet_free(&packet); avcodec_free_context(&codec_context); avformat_close_input(&format_context); if (io_context->buffer != nullptr) { av_free(io_context->buffer); } avio_context_free(&io_context); } static int read_packet(void *opaque, uint8_t *buf, int buf_size) { auto state = static_cast(opaque); // emscripten_console_log( // ("read packet, offset = " + std::to_string(state->data_pos) + // " buf_size = " + std::to_string(buf_size)) // .c_str()); auto data_size = state->data->size(); if (state->data_pos >= data_size) { if (state->is_streaming) { return AVERROR(EAGAIN); } else { return AVERROR_EOF; } } buf_size = std::min(buf_size, static_cast(data_size - state->data_pos)); if (buf_size <= 0) { if (state->is_streaming) { return AVERROR(EAGAIN); } else { return AVERROR_EOF; } } int bytes_read = state->data->read(state->data_pos, buf_size, buf); state->data_pos += bytes_read; return bytes_read; } static int64_t seek_packet(void *opaque, int64_t offset, int whence) { auto state = static_cast(opaque); auto data_size = state->data->size(); if (whence & AVSEEK_SIZE) { return data_size; } if (whence == SEEK_SET) { if (offset < 0) { return AVERROR(EINVAL); } state->data_pos = std::min(static_cast(offset), data_size); return offset; } if (whence == SEEK_CUR) { if (offset < 0 && static_cast(-offset) > state->data_pos) { return AVERROR(EINVAL); } state->data_pos = std::min( static_cast(static_cast(state->data_pos) + offset), data_size); return state->data_pos; } if (whence == SEEK_END) { if (offset < 0 && static_cast(-offset) > data_size) { return AVERROR(EINVAL); } state->data_pos = std::min( static_cast(static_cast(data_size) + offset), data_size); return state->data_pos; } return AVERROR(EINVAL); } bool seek(int frameOffset) { if (partially_consumed_offset >= 0) { av_freep(&resampled_data); partially_consumed_offset = -1; } if (have_packet) { av_packet_unref(packet); have_packet = false; } // Account for preroll (aka preskip) const auto preroll = format_context->streams[audio_stream_index]->codecpar->seek_preroll; const auto stream_timebase = format_context->streams[audio_stream_index]->time_base; frameOffset -= preroll; // Convert sample offset to stream time base int64_t seek_timestamp = av_rescale_q(frameOffset, {1, sample_rate}, stream_timebase); int res = avformat_seek_file(format_context, audio_stream_index, INT64_MIN, seek_timestamp, seek_timestamp, 0); if (res < 0) { return false; } avcodec_flush_buffers(codec_context); for (;;) { if (!nextFrame()) { return false; } auto frame_start_sample = av_rescale_q(frame->pts, stream_timebase, {1, sample_rate}) - zero_pts_frame_index; auto frame_end_sample = frame_start_sample + frame->nb_samples; if (frame_end_sample <= frameOffset + preroll) { continue; } if (frame_start_sample > frameOffset + preroll) { return false; } partially_consumed_offset = frameOffset + preroll - frame_start_sample; resampleCurrentFrame(); return true; } return false; // unreachable } void resampleCurrentFrame() { av_samples_alloc((uint8_t **)&resampled_data, nullptr, channel_count, frame->nb_samples, AV_SAMPLE_FMT_FLT, 0); int samples_converted = swr_convert(resampler, (uint8_t **)&resampled_data, frame->nb_samples, (const uint8_t **)frame->data, frame->nb_samples); assert(samples_converted == frame->nb_samples); } int readSome(BufferF32 output) { assert(output.getChannelCount() == channel_count); output.fill(0); int framesReadToOutput = 0; while (framesReadToOutput < output.getFrameCount()) { if (partially_consumed_offset >= 0) { int framesToRead = std::min( static_cast(output.getFrameCount()) - framesReadToOutput, frame->nb_samples - partially_consumed_offset); auto readBuf = resampled_data + channel_count * partially_consumed_offset; for (int ch = 0; ch < output.getChannelCount(); ch++) { auto writeBuf = output.getChannelData(ch) + framesReadToOutput; for (int i = 0; i < framesToRead; i++) { writeBuf[i] = readBuf[channel_count * i + ch]; } } framesReadToOutput += framesToRead; partially_consumed_offset += framesToRead; if (partially_consumed_offset >= frame->nb_samples) { partially_consumed_offset = -1; av_freep(&resampled_data); } continue; } if (!nextFrame()) { return framesReadToOutput; } partially_consumed_offset = 0; resampleCurrentFrame(); } return framesReadToOutput; } }; FfmpegAudioBuffer::FfmpegAudioBuffer(int token, float streamReadOffset) : token(token), nextFrameOffset(0), streamReadOffset(streamReadOffset), ffmpegState(nullptr), createdOnMainThread(emscripten_is_main_browser_thread()), completedBlankPromise(make_blank_promise()) {} void FfmpegAudioBuffer::reset() { ffmpegState.reset(); data.reset(); nextFrameOffset = 0; } void FfmpegAudioBuffer::setData(const std::shared_ptr &data) { reset(); this->data = data; ffmpegState = std::make_unique(data); } void FfmpegAudioBuffer::resolvePromise( const std::shared_ptr &buffer) { if (createdOnMainThread && emscripten_is_main_browser_thread()) { completedBlankPromise["capturedResolve"](buffer); } } FfmpegAudioBufferPromise FfmpegAudioBuffer::getCompletedPromise() const { return completedBlankPromise["promise"].as(); } FfmpegAudioBuffer::~FfmpegAudioBuffer() { reset(); } std::shared_ptr FfmpegAudioBuffer::createFromUrl(const std::string &url, const std::optional &uuid) { auto buffer = std::make_shared( 0); // Temporary token, will be updated int token = ffmpegLibrary.getDataOrWait(url, uuid, buffer); buffer->token = token; // Update with the actual token // Check if we need to start a download (data wasn't immediately available) if (!buffer->data) { emscripten_fetch_attr_t attr; emscripten_fetch_attr_init(&attr); strcpy(attr.requestMethod, "GET"); attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; attr.onsuccess = downloadSucceeded; attr.onerror = downloadFailed; emscripten_fetch(&attr, url.c_str()); } return buffer; } // clang-format off EM_JS(void, _stream_to_buffer, (const char *url, emscripten::EM_VAL aobSourceHandle), { const urlString = UTF8ToString(url); const aobSource = Emval.toValue(aobSourceHandle); (async() => { try { const response = await fetch(urlString); console.log('Streaming buffer: Fetched url ' + urlString + ' with status ' + response.status); if(!response.ok) { throw new Error('Streaming buffer: Failed to fetch url ' + urlString + ' with status ' + response.status); } const reader = response.body?.getReader(); if (!reader) { throw new Error('Streaming buffer: Failed to get reader for url ' + urlString); } try { while (true) { const { value, done } = await reader.read(); if (done) { break; } aobSource.write(value); } } finally { reader.releaseLock(); } } finally { aobSource.setCompleted(); aobSource.delete(); } })(); }); // clang-format on std::shared_ptr FfmpegAudioBuffer::createFromUrlStreaming( const std::string &url, const std::optional &uuid, std::optional streamReadOffset) { // 0 is a temporary token, will be updated // default to 5ms late reads to compensate for the limiter in audiopipe auto buffer = std::make_shared(0, streamReadOffset.value_or(0.005f)); int token = ffmpegLibrary.getDataOrWait(url, uuid, buffer); buffer->token = token; // Update with the actual token // Check if we need to start a download (data wasn't immediately available) if (!buffer->data) { auto aobSource = std::make_shared(); _stream_to_buffer(url.c_str(), emscripten::val(aobSource).as_handle()); ffmpegLibrary.setData(url, aobSource); } return buffer; } std::shared_ptr FfmpegAudioBuffer::createFromByteArray(const FfmpegByteArrayType &source, const std::optional &uuid) { const auto sourceLength = source["length"].as(); std::vector data(sourceLength); auto moduleHeapBuffer = emscripten::val::module_property("GROWABLE_HEAP_U8")()["buffer"]; auto thisData = emscripten::val::global("Uint8Array") .new_(moduleHeapBuffer, reinterpret_cast(data.data()), sourceLength); thisData.call("set", source.as()); auto dataSource = std::make_shared(std::move(data)); auto buffer = std::make_shared(0); buffer->setData(dataSource); // If UUID is provided, register the buffer so it can be looked up later if (uuid.has_value()) { int token = ffmpegLibrary.registerDataWithUuid(uuid.value(), dataSource); buffer->token = token; } return buffer; } std::shared_ptr FfmpegAudioBuffer::createFromReadCallback( size_t size, FfmpegReadCallbackType readCallback, FfmpegCloseCallbackType closeCallback) { auto buffer = std::make_shared(0); buffer->setData( std::make_shared(size, readCallback, closeCallback)); return buffer; } std::shared_ptr FfmpegAudioBuffer::createFromUuid(const std::string &uuid) { auto buffer = std::make_shared(0); if (ffmpegLibrary.getUuidDataOrWait(uuid, buffer)) { return buffer; } return nullptr; } bool FfmpegAudioBuffer::addUuidAlias(const std::string &existingUuid, const std::string &newUuid) { return ffmpegLibrary.addUuidAlias(existingUuid, newUuid); } int FfmpegAudioBuffer::getSampleRate() const { return ffmpegState ? ffmpegState->sample_rate : 48000; } int FfmpegAudioBuffer::getChannelCount() const { return ffmpegState ? ffmpegState->channel_count : 2; } int FfmpegAudioBuffer::getFrameCount() const { if (!ffmpegState) { return 0; } if (ffmpegState->is_streaming) { // duplicate logic because state may not be initialized yet return INT_MAX; } return ffmpegState->frame_count; } std::shared_ptr FfmpegAudioBuffer::createFromByteVector(const std::vector &source) { auto buffer = std::make_shared(0); // No token for direct data buffer->setData(std::make_shared(source)); return buffer; } void FfmpegAudioBuffer::read(int frameOffset, BufferF32 output) { frameOffset += streamReadOffset * getSampleRate(); if (frameOffset < 0) { const auto skipFrames = -frameOffset; output.fill(0); if (skipFrames < output.getFrameCount()) { auto sliced = output.slice(skipFrames); read(0, sliced); } return; } if (ffmpegState && ffmpegState->is_streaming && (!ffmpegState->initialized || ffmpegState->read_frame_err)) { // Try to reinitialize the state ffmpegState = std::make_unique(data); nextFrameOffset = 0; } if (!ffmpegState || !ffmpegState->initialized) { output.fill(0); return; } if (frameOffset != nextFrameOffset) { if (!ffmpegState->seek(frameOffset)) { if (!ffmpegState->is_streaming) { #ifdef BUILD_TESTS assert(false); #endif } output.fill(0); nextFrameOffset = -1; // indicating we're at an invalid position return; } nextFrameOffset = frameOffset; } int framesRead = ffmpegState->readSome(output); for (int channel = 0; channel < output.getChannelCount(); channel++) { auto channelData = output.getChannelData(channel); vfastnanzero_array(channelData, channelData, std::min(framesRead, (int)output.getFrameCount())); } if (framesRead < output.getFrameCount()) { output.slice(framesRead).fill(0); } nextFrameOffset = frameOffset + framesRead; } std::shared_ptr FfmpegAudioBuffer::createClone() const { auto buffer = std::make_shared(token); if (data) { // share the same data buffer->setData(data); return buffer; } // otherwise, register to be notified when the data is available if (token != 0) { ffmpegLibrary.getTokenDataOrWait(token, buffer); } return buffer; } bool FfmpegAudioBuffer::isCompleted() const { if (!data) { return false; } return data->isCompleted(); } float FfmpegAudioBuffer::getAvailableDuration() const { if (!ffmpegState) { return 0.f; } if (ffmpegState->is_streaming) { const auto dataAOB = dynamic_cast(data.get()); if (dataAOB) { return dataAOB->getDecodedLength(); } return 0.f; } return (float)ffmpegState->frame_count / (float)ffmpegState->sample_rate; } EM_JS(char *, decodeToPcmTestHelper, (const char *str), { if (typeof global != 'undefined' && typeof global._decodeToPcmTestHelper != 'undefined') { return stringToNewUTF8(global._decodeToPcmTestHelper(UTF8ToString(str))); } return stringToNewUTF8('missing global._decodeToPcmTestHelper'); }); static std::string callDecodeToPcmTestHelper(const std::string &filename) { char *res_cstr = decodeToPcmTestHelper(filename.c_str()); std::string res(res_cstr); free(res_cstr); return res; } EMSCRIPTEN_BINDINGS(ffmpegbuffer) { using namespace emscripten; register_type("Promise"); register_type("Uint8Array"); register_type( "(buf: Uint8Array, offset: number) => void"); register_type("() => void"); register_optional(); register_optional(); class_>( "FfmpegAudioBuffer") .smart_ptr>("FfmpegAudioBuffer") .class_function("createFromUrl(url, uuid)", &FfmpegAudioBuffer::createFromUrl, nonnull()) .class_function("createFromUrlStreaming(url, uuid, streamReadOffset)", &FfmpegAudioBuffer::createFromUrlStreaming, nonnull()) .class_function("createFromUuid(uuid)", &FfmpegAudioBuffer::createFromUuid) .class_function("createFromByteArray(source, uuid)", &FfmpegAudioBuffer::createFromByteArray, nonnull()) .class_function("addUuidAlias(existingUuid, newUuid)", &FfmpegAudioBuffer::addUuidAlias) .class_function( "createFromReadCallback(size, readCallback, closeCallback)", &FfmpegAudioBuffer::createFromReadCallback, nonnull()) .function("createClone", &FfmpegAudioBuffer::createClone, nonnull()) .property("promise", &FfmpegAudioBuffer::getCompletedPromise) .property("nextFrameOffset", &FfmpegAudioBuffer::getNextFrameOffset) .property("completed", &FfmpegAudioBuffer::isCompleted) .property("availableDuration", &FfmpegAudioBuffer::getAvailableDuration); class_("AOBDataSource") .function("write", &AOBDataSource::write) .function("setCompleted", &AOBDataSource::setCompleted) .smart_ptr>("AOBDataSource"); } TEST_CASE("FfmpegAudioBuffer aligned decoding", "[ffmpegbuffer]") { for (auto basename : {"random_spontaneous_devotion.opus", "test_progressive.m4a", "random_spontaneous_devotion.m4a"}) { DYNAMIC_SECTION("Aligned decoding of " << basename) { const std::string test_path = "testdata/" + std::string{basename}; std::string expected_pcm_path = callDecodeToPcmTestHelper(test_path); std::ifstream test_file(test_path, std::ios::binary); std::vector test_data( (std::istreambuf_iterator(test_file)), std::istreambuf_iterator()); std::ifstream expected_file(expected_pcm_path, std::ios::binary); std::ofstream dump_file; if (false) { dump_file.open("dump.pcm", std::ios::binary); } REQUIRE(test_file.is_open()); REQUIRE(expected_file.is_open()); // Read the expected PCM file (f32le, interleaved) into a planar float // buffer std::vector expected_interleaved( (std::istreambuf_iterator(expected_file)), std::istreambuf_iterator()); // Convert bytes to float expected_file.seekg(0, std::ios::end); const size_t num_floats = expected_file.tellg() / sizeof(float); expected_file.seekg(0, std::ios::beg); expected_interleaved.resize(num_floats); expected_file.read(reinterpret_cast(expected_interleaved.data()), num_floats * sizeof(float)); const int channel_count = 2; const int frame_count = num_floats / channel_count; BufferF32 expected_buffer(channel_count, frame_count); for (int frame = 0; frame < frame_count; ++frame) { for (int ch = 0; ch < channel_count; ++ch) { expected_buffer.getChannelData(ch)[frame] = expected_interleaved[frame * channel_count + ch]; } } auto buffer = FfmpegAudioBuffer::createFromByteVector(test_data); auto output = BufferF32(channel_count, 4096); for (int offset : {0, 123456, 234567, 123, 4940000, 454545, 0, 999999}) { if (dump_file.is_open()) { dump_file.seekp(0, std::ios::beg); for (int i = 0; i < 2 * offset; i++) { float zero = 0.f; dump_file.write(reinterpret_cast(&zero), sizeof(float)); } } emscripten_console_log( ("testing offset = " + std::to_string(offset)).c_str()); for (int block = 0; block < 100; ++block) { buffer->read(offset + block * output.getFrameCount(), output); bool allclose = true; for (int frame = 0; frame < output.getFrameCount() && allclose; ++frame) { for (int ch = 0; ch < output.getChannelCount() && allclose; ++ch) { if (dump_file.is_open()) { dump_file.write( reinterpret_cast(output.getChannelData(ch) + frame), sizeof(float)); } if (block < 10) { // opus takes a while to fully converge... continue; } const float expected = expected_buffer.getChannelData( ch)[frame + offset + block * output.getFrameCount()]; const float actual = output.getChannelData(ch)[frame]; if (std::fabs(actual - expected) > std::max(0.01f * std::max(std::fabs(expected), std::fabs(actual)), 1e-4f)) { emscripten_console_log( ("offset " + std::to_string(offset) + " block " + std::to_string(block) + " frame " + std::to_string(frame) + " ch " + std::to_string(ch) + " expected " + std::to_string(expected) + " actual " + std::to_string(actual)) .c_str()); allclose = false; break; } } } REQUIRE(allclose); } } } } } TEST_CASE("FfmpegAudioBuffer decoder performance", "[.][ffmpegbuffer_perf]") { std::ifstream test_file("testdata/random_spontaneous_devotion.m4a", std::ios::binary); std::vector test_data((std::istreambuf_iterator(test_file)), std::istreambuf_iterator()); REQUIRE(test_file.is_open()); for (int round = 0; round < 3; ++round) { auto buffer = FfmpegAudioBuffer::createFromByteVector(test_data); auto output = BufferF32(buffer->getChannelCount(), 4096); auto start = std::chrono::high_resolution_clock::now(); for (int i = 0; i < buffer->getFrameCount(); i += output.getFrameCount()) { buffer->read(i, output); } auto end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end - start); emscripten_console_log( ("Time taken: " + std::to_string(duration.count()) + " milliseconds") .c_str()); } } TEST_CASE("FfmpegAudioBuffer streaming decode behavior", "[ffmpegbuffer_streaming]") { // Test with opus file (assuming 128kbps bitrate) const std::string test_path = "testdata/progressive.webm"; std::string expected_pcm_path = callDecodeToPcmTestHelper(test_path); // Load test file data and expected PCM std::ifstream test_file(test_path, std::ios::binary); std::ifstream expected_file(expected_pcm_path, std::ios::binary); REQUIRE(test_file.is_open()); REQUIRE(expected_file.is_open()); std::vector full_data((std::istreambuf_iterator(test_file)), std::istreambuf_iterator()); // Read expected PCM data std::vector expected_interleaved( (std::istreambuf_iterator(expected_file)), std::istreambuf_iterator()); expected_file.seekg(0, std::ios::end); const size_t num_floats = expected_file.tellg() / sizeof(float); expected_file.seekg(0, std::ios::beg); expected_interleaved.resize(num_floats); expected_file.read(reinterpret_cast(expected_interleaved.data()), num_floats * sizeof(float)); const int channel_count = 2; const int expected_frame_count = num_floats / channel_count; BufferF32 expected_buffer(channel_count, expected_frame_count); for (int frame = 0; frame < expected_frame_count; ++frame) { for (int ch = 0; ch < channel_count; ++ch) { expected_buffer.getChannelData(ch)[frame] = expected_interleaved[frame * channel_count + ch]; } } auto aobSource = std::make_shared(); auto buffer = std::make_shared(0); buffer->setData(aobSource); SECTION("Frame count returns INT_MAX for streaming") { // Add enough data to initialize const size_t init_chunk = 16384; if (full_data.size() > init_chunk) { std::vector first_chunk(full_data.begin(), full_data.begin() + init_chunk); aobSource->data.append(first_chunk.size(), first_chunk.data()); if (buffer->getChannelCount() > 0) { REQUIRE(buffer->getFrameCount() == INT_MAX); } } } SECTION("Progressive streaming with PCM validation") { // Calculate bytes needed for specific durations (128kbps = 16KB/s) const int sample_rate = 48000; const double bitrate_bytes_per_second = 128000.0 / 8.0; // 128kbps to bytes/s const size_t extra_buffer = 128 * 1024; // Extra 128KB as requested // Test reading first 2 seconds of audio const double target_seconds = 2.0; const int target_frames = static_cast(target_seconds * sample_rate); const size_t bytes_needed = static_cast(target_seconds * bitrate_bytes_per_second) + extra_buffer; REQUIRE(full_data.size() > bytes_needed); REQUIRE(expected_frame_count > target_frames); // Feed calculated amount of data std::vector partial_data( full_data.begin(), full_data.begin() + std::min(bytes_needed, full_data.size())); aobSource->data.append(partial_data.size(), partial_data.data()); // Read and compare first portion auto output = BufferF32(channel_count, 4096); int frames_validated = 0; const int max_validation_frames = std::min(target_frames - 10 * 4096, expected_frame_count - 10 * 4096); for (int offset = 0; offset < max_validation_frames && frames_validated < target_frames / 2; offset += output.getFrameCount()) { buffer->read(offset, output); // Compare against expected (skip first few blocks for opus convergence) if (offset >= 10 * output.getFrameCount()) { bool matches = true; for (int frame = 0; frame < output.getFrameCount() && matches; ++frame) { const int expected_frame_idx = offset + frame; if (expected_frame_idx >= expected_frame_count) break; for (int ch = 0; ch < output.getChannelCount() && matches; ++ch) { const float expected = expected_buffer.getChannelData(ch)[expected_frame_idx]; const float actual = output.getChannelData(ch)[frame]; const float tolerance = std::max( 0.01f * std::max(std::fabs(expected), std::fabs(actual)), 1e-4f); if (std::fabs(actual - expected) > tolerance) { matches = false; } } } REQUIRE(matches); frames_validated += output.getFrameCount(); } } } SECTION("Reading beyond available data returns silence") { // Add data for ~1 second (128kbps) const double available_seconds = 1.0; const size_t bytes_for_duration = static_cast(available_seconds * 128000.0 / 8.0); REQUIRE(full_data.size() > bytes_for_duration); std::vector limited_data(full_data.begin(), full_data.begin() + bytes_for_duration); aobSource->data.append(limited_data.size(), limited_data.data()); // Try to read from well beyond available data const int beyond_available_offset = static_cast( available_seconds * 48000 * 2); // 2x the available duration auto output = BufferF32(2, 4096); buffer->read(beyond_available_offset, output); // Should return silence (all zeros) for data beyond what's available bool all_silence = true; for (int ch = 0; ch < output.getChannelCount(); ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-6f) { all_silence = false; break; } } if (!all_silence) break; } REQUIRE(all_silence); } SECTION("Incremental data feeding with validation") { // Feed data in chunks and validate progressive decoding const size_t chunk_size = 8192; const int sample_rate = 48000; auto output = BufferF32(channel_count, 1024); size_t bytes_fed = 0; int last_successful_read_offset = 0; for (size_t chunk_start = 0; chunk_start < full_data.size() && chunk_start < 64 * 1024; chunk_start += chunk_size) { size_t chunk_end = std::min(chunk_start + chunk_size, full_data.size()); std::vector chunk(full_data.begin() + chunk_start, full_data.begin() + chunk_end); aobSource->data.append(chunk.size(), chunk.data()); bytes_fed += chunk.size(); // Estimate how much audio should be available (conservative estimate) const double estimated_seconds = (bytes_fed * 8.0) / (128000.0 * 1.5); // Use 1.5x for safety margin const int estimated_frames = static_cast(estimated_seconds * sample_rate); // Try to read from current position if (estimated_frames > last_successful_read_offset + output.getFrameCount()) { buffer->read(last_successful_read_offset, output); // Verify we got some non-zero data (after initial convergence period) if (bytes_fed > 32768) { // After feeding enough data bool has_audio = false; for (int ch = 0; ch < output.getChannelCount() && !has_audio; ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-6f) { has_audio = true; break; } } } REQUIRE(has_audio); // Should have some audio content last_successful_read_offset += output.getFrameCount(); } } } } SECTION("Seek beyond available data, then receive more data") { // Phase 1: Feed initial data until we can decode a non-silent frame const double initial_seconds = 2.0; const size_t initial_bytes = static_cast(initial_seconds * 128000.0 / 8.0); emscripten_console_log( ("initial_bytes: " + std::to_string(initial_bytes)).c_str()); REQUIRE(full_data.size() > initial_bytes * 3); std::vector initial_data(full_data.begin(), full_data.begin() + initial_bytes); aobSource->data.append(initial_data.size(), initial_data.data()); // Phase 2: Begin decoding in available region (first 1 second) const int sample_rate = 48000; auto output = BufferF32(channel_count, 4096); const int initial_read_offset = static_cast(0.5 * sample_rate); // 0.5 seconds in buffer->read(initial_read_offset, output); // Verify we got some audio content in the initial region bool has_initial_audio = false; for (int ch = 0; ch < output.getChannelCount() && !has_initial_audio; ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-6f) { has_initial_audio = true; break; } } } REQUIRE(has_initial_audio); // Phase 3: Seek forward to region beyond available data (4 seconds in) const int beyond_available_offset = static_cast(4.0 * sample_rate); buffer->read(beyond_available_offset, output); // Should get silence since we don't have data for 4 seconds yet bool is_silence = true; for (int ch = 0; ch < output.getChannelCount(); ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-6f) { is_silence = false; break; } } if (!is_silence) break; } REQUIRE(is_silence); // Phase 4: Read a few more buffers of silence to track our position const int silence_buffers_read = 3; for (int i = 0; i < silence_buffers_read; i++) { buffer->read(beyond_available_offset + (i + 1) * output.getFrameCount(), output); // Verify still silence bool still_silence = true; for (int ch = 0; ch < output.getChannelCount(); ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-6f) { still_silence = false; break; } } if (!still_silence) break; } REQUIRE(still_silence); } // Phase 5: Feed more data to cover the region we're trying to read const double total_needed_seconds = 5.0; // Need to cover up to 5 seconds const size_t total_bytes_needed = static_cast(total_needed_seconds * 128000.0 / 8.0) + 128 * 1024; const size_t additional_bytes = total_bytes_needed - initial_bytes; REQUIRE(full_data.size() > total_bytes_needed); std::vector additional_data( full_data.begin() + initial_bytes, full_data.begin() + std::min(total_bytes_needed, full_data.size())); aobSource->data.append(additional_data.size(), additional_data.data()); // Phase 6: Continue reading from where we left off // We should now get non-silent audio at the position we were seeking to const int current_read_position = beyond_available_offset + silence_buffers_read * output.getFrameCount(); buffer->read(current_read_position, output); // Compare against expected PCM data at this position bool has_expected_audio = false; REQUIRE(current_read_position < expected_frame_count); // Allow for some convergence time after the data becomes available for (int convergence_attempt = 0; convergence_attempt < 5; convergence_attempt++) { const int read_pos = current_read_position + convergence_attempt * output.getFrameCount(); if (read_pos >= expected_frame_count) break; buffer->read(read_pos, output); // Check if we're getting the expected audio content bool matches_expected = true; int matching_frames = 0; for (int frame = 0; frame < output.getFrameCount() && matches_expected; ++frame) { const int expected_frame_idx = read_pos + frame; if (expected_frame_idx >= expected_frame_count) break; for (int ch = 0; ch < output.getChannelCount(); ++ch) { const float expected = expected_buffer.getChannelData(ch)[expected_frame_idx]; const float actual = output.getChannelData(ch)[frame]; const float tolerance = std::max( 0.02f * std::max(std::fabs(expected), std::fabs(actual)), 2e-4f); if (std::fabs(actual - expected) <= tolerance) { matching_frames++; } else if (std::fabs(actual) > 1e-6f || std::fabs(expected) > 1e-6f) { // Only count as mismatch if at least one value is non-zero matches_expected = false; break; } } } // If we got mostly matching frames or clearly non-silent audio, // we're good if (matches_expected && matching_frames > output.getFrameCount() / 2) { has_expected_audio = true; break; } // Or if we at least got non-silent audio (even if not perfectly // matching) bool has_audio = false; for (int ch = 0; ch < output.getChannelCount() && !has_audio; ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-5f) { has_audio = true; break; } } } if (has_audio) { has_expected_audio = true; break; } } REQUIRE(has_expected_audio); // Phase 7: Verify we can seek back to the original available region buffer->read(initial_read_offset, output); bool has_audio_after_seek_back = false; for (int ch = 0; ch < output.getChannelCount() && !has_audio_after_seek_back; ch++) { for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(output.getChannelData(ch)[frame]) > 1e-6f) { has_audio_after_seek_back = true; break; } } } REQUIRE(has_audio_after_seek_back); } SECTION( "Ffmpeg streaming init is crash-free for any number of initial bytes") { const size_t chunk_size = 13; const size_t total_bytes = 32768; REQUIRE(full_data.size() >= total_bytes + chunk_size); auto output = BufferF32(channel_count, 4096); output.fill(0); bool has_audio = false; for (size_t chunk_start = 0; chunk_start < total_bytes && !has_audio; chunk_start += chunk_size) { aobSource->data.append(chunk_size, full_data.data() + chunk_start); buffer->read(0, output); auto bufferData = output.getChannelData(0); for (int frame = 0; frame < output.getFrameCount(); frame++) { if (std::fabs(bufferData[frame]) > 1e-6f) { has_audio = true; break; } } } REQUIRE(has_audio); } }