#include "audioclip.h" #include "buffer.h" #include "randomaccessaudioreadable.h" #include "remotebuffer.h" #include "resampler.h" #include "simd.h" #include "timestretch.h" #include "timing/arrangement.h" #include "timing/arrangement_offset.h" #include "timing/composition.h" #include "timing/time_units.h" #include #include #include #include AudioClip::AudioClip( std::shared_ptr underlyingReadable, std::shared_ptr> warpMap, int playbackSampleRate, float gain, double timelineStartBeats, double timelineEndBeats, double loopStartBeats, double loopEndBeats, double readStartBeats, double fadeInBeats, double fadeInExponent, double fadeOutBeats, double fadeOutExponent, float transposition, double warpedContentBps, bool warpEnabled, const Uuid &arrangementId) : originalUnderlyingReadable(std::move(underlyingReadable)), playbackSampleRate(playbackSampleRate), previousUnderlyingReadOffset(0), previousRequestedPosition(-INFINITY), warpedContentBps(warpedContentBps), warpEnabled(warpEnabled), microfadeMemory(2, 16), microfadeOffset(16), transposition(transposition), gain(gain), timelineStartBeats(timelineStartBeats), timelineEndBeats(timelineEndBeats), loopStartBeats(loopStartBeats), loopEndBeats(loopEndBeats), readStartBeats(readStartBeats), fadeInBeats(fadeInBeats), fadeInExponent(fadeInExponent), fadeOutBeats(fadeOutBeats), fadeOutExponent(fadeOutExponent), arrangementId(arrangementId), unwarpedExpectedReadOffset(INT_MIN), warpMap(std::move(warpMap)) { assert(playbackSampleRate > 0); assert(std::isfinite(gain)); assert(timelineStartBeats <= timelineEndBeats); assert(loopStartBeats < loopEndBeats); assert(std::isfinite(readStartBeats)); assert(!std::isnan(fadeInBeats) && !std::isnan(fadeOutBeats)); assert(std::isfinite(fadeInExponent) && std::isfinite(fadeOutExponent)); assert(std::isfinite(transposition)); assert(std::isfinite(warpedContentBps)); } void AudioClip::readSegmentWarped( const std::shared_ptr ×tretchReader, const RenderContext *renderContext, BufferF32 output, float transposition, double adjustedReadStartBeats, double adjustedTimelineStartBeats, AudioClip::AnalyticsResult &analyticsResult) { using namespace time_units; using namespace time_transform; const auto offset_map = ArrangementBeatOffsetMap( Beats(adjustedTimelineStartBeats), Beats(timelineEndBeats)); const auto loop_map = ArrangementBeatLoopMap(Beats(adjustedReadStartBeats), Beats(loopStartBeats), Beats(loopEndBeats)); const auto offset_loop_map = CompositionMap(offset_map, loop_map); const auto segment_tempo_map = TimelineTempoMap(renderContext->bps); const auto full_map = CompositionMap(offset_loop_map, segment_tempo_map); boundedSubdivide( [&](const time_transform::MappedSegment< time_units::BeatsTag, time_units::SecondsTag, double> &segment, time_units::Beats position, time_units::Seconds mappedPosition, size_t offset, size_t frameCount, time_units::BeatsDelta segmentDuration) -> std::optional> { if (!segment.has_mark( time_transform::SegmentMarks::INVALID_ARRANGEMENT_TIME)) { analyticsResult.didPlayContent = true; auto outputSlice = output.slice(offset, offset + frameCount); auto localPositionIt = offset_loop_map.getSegmentIteratorAt(position); assert(localPositionIt != offset_loop_map.end()); auto localPosition = localPositionIt->map_point(position); assert(localPosition.has_value()); RenderContext localRenderContext = *renderContext; localRenderContext.position = position; localRenderContext.duration = segmentDuration; analyticsResult.didLoop |= timestretchReader->readSegment( &localRenderContext, *localPosition, transposition, outputSlice); } return std::nullopt; }, full_map, renderContext->position, playbackSampleRate, output.getFrameCount()); analyticsResult.lastPosition = timestretchReader->getLastAnalysisPosition(); } template inline time_units::Seconds projectBeatsToSeconds(const MapT &map, time_units::Beats beats) { auto it = map.getSegmentIteratorAt(beats); assert(it != map.end()); auto mappedPosition = it->map_point(beats); assert(mappedPosition.has_value()); return mappedPosition.value(); } void AudioClip::readSegmentUnwarped( const std::shared_ptr &underlyingReadable, const RenderContext *renderContext, BufferF32 output, double adjustedReadStartBeats, double adjustedTimelineStartBeats, AudioClip::AnalyticsResult &analyticsResult) { using namespace time_units; using namespace time_transform; const auto adjustedTimelineStartSeconds = projectBeatsToSeconds( renderContext->globalBeatsToSeconds, Beats(adjustedTimelineStartBeats)); const auto timelineEndSeconds = projectBeatsToSeconds( renderContext->globalBeatsToSeconds, Beats(timelineEndBeats)); const auto adjustedReadStartSeconds = projectBeatsToSeconds(*warpMap, Beats(adjustedReadStartBeats)); const auto loopStartSeconds = projectBeatsToSeconds(*warpMap, Beats(loopStartBeats)); const auto loopEndSeconds = projectBeatsToSeconds(*warpMap, Beats(loopEndBeats)); const auto offset_map = ArrangementOffsetMap( adjustedTimelineStartSeconds, timelineEndSeconds); const auto loop_map = ArrangementLoopMap( adjustedReadStartSeconds, loopStartSeconds, loopEndSeconds); const auto offset_loop_map = CompositionMap(offset_map, loop_map); const auto full_map = CompositionMap(renderContext->globalBeatsToSeconds, offset_loop_map); auto analyticsLastPosition = Seconds(0.0); boundedSubdivide( [&](const time_transform::MappedSegment< time_units::BeatsTag, time_units::SecondsTag, double> &segment, time_units::Beats position, time_units::Seconds mappedPosition, size_t offset, size_t frameCount, time_units::BeatsDelta segmentDuration) -> std::optional> { if (!segment.has_mark( time_transform::SegmentMarks::INVALID_ARRANGEMENT_TIME)) { analyticsResult.didPlayContent = true; auto outputSlice = output.slice(offset, offset + frameCount); const int readOffset = std::round(mappedPosition.raw() * playbackSampleRate); underlyingReadable->readZeroPadded(readOffset, outputSlice); if (unwarpedExpectedReadOffset != INT_MIN && readOffset != unwarpedExpectedReadOffset) { analyticsResult.didLoop = true; } unwarpedExpectedReadOffset = readOffset + outputSlice.getFrameCount(); analyticsLastPosition = mappedPosition + SecondsDelta((double)frameCount / (double)playbackSampleRate); } return std::nullopt; }, full_map, renderContext->position, playbackSampleRate, output.getFrameCount()); analyticsResult.lastPosition = analyticsLastPosition; } void AudioClip::readSegment(const RenderContext *renderContext, BufferF32 output) { using namespace time_units; using namespace time_transform; // Special handling for readStartBeats outside of loop auto adjustedReadStartBeats = readStartBeats; auto adjustedTimelineStartBeats = timelineStartBeats; if (readStartBeats < loopStartBeats) { const auto beatsBeforeLoop = loopStartBeats - readStartBeats; adjustedReadStartBeats = loopStartBeats; adjustedTimelineStartBeats = std::min(timelineStartBeats + beatsBeforeLoop, timelineEndBeats); } else if (readStartBeats > loopEndBeats) { // don't know what this would sound like return; } if (adjustedTimelineStartBeats == timelineEndBeats) { // no point in reading anything return; } AnalyticsResult analyticsResult{false, false, Seconds(0.0)}; if (backend == nullptr) { return; } const auto &underlyingReadable = backend->underlyingReadable; if (underlyingReadable == nullptr) { return; } const auto currentTransposition = getTransposition(); if (warpEnabled) { const auto ×tretchReader = backend->timestretchReader; if (timestretchReader == nullptr) { return; } if (timestretchReader->acquire(this)) { timestretchReader->setUnderlying(backend->underlyingReadable, warpMap); } readSegmentWarped(timestretchReader, renderContext, output, currentTransposition, adjustedReadStartBeats, adjustedTimelineStartBeats, analyticsResult); } else { const auto &resampler = backend->resampler; if (resampler == nullptr) { return; } const auto underlyingSampleRate = underlyingReadable->getSampleRate(); std::shared_ptr useReadable = nullptr; if (currentTransposition != 0.0 || playbackSampleRate != underlyingReadable->getSampleRate()) { const auto readableSampleRateForTransposition = std::round(100.0 * (double)underlyingSampleRate * std::pow(2.0, currentTransposition / 12.0)); const int denominator = 100 * playbackSampleRate; if (resampler->acquire(this)) { resampler->setUnderlying(underlyingReadable, readableSampleRateForTransposition, denominator); } else if (resampler->getNumerator() != readableSampleRateForTransposition || resampler->getDenominator() != denominator) { resampler->setSampleRateRatio(readableSampleRateForTransposition, denominator); } useReadable = resampler; } else { useReadable = underlyingReadable; } readSegmentUnwarped(useReadable, renderContext, output, adjustedReadStartBeats, adjustedTimelineStartBeats, analyticsResult); } if (renderContext->analyticsObserver != nullptr && analyticsResult.didPlayContent) { if (analyticsResult.didLoop) { renderContext->analyticsObserver->trackLooped(arrangementId); } renderContext->analyticsObserver->trackPlayback( arrangementId, analyticsResult.lastPosition); } applyFadesAndGain(renderContext->position.raw(), renderContext->bps, output); } void AudioClip::applyFadesAndGain(double position, double bps, BufferF32 output) { const auto projectSamplesToBeats = bps / (double)playbackSampleRate; const auto outputChannelCount = output.getChannelCount(); const auto happyFadeInBeats = std::abs(fadeInBeats) + 1.0e-6f; const auto happyFadeOutBeats = std::abs(fadeOutBeats) + 1.0e-6f; const auto happyFadeInExponent = std::max(0.0, fadeInExponent); const auto happyFadeOutExponent = std::max(0.0, fadeOutExponent); float sampleBeats[output.getFrameCount()]; float gainCurve[output.getFrameCount()]; #pragma clang loop vectorize(enable) for (int i = 0; i < output.getFrameCount(); i++) { sampleBeats[i] = position + i * projectSamplesToBeats; } vfast_curve_fade_array(sampleBeats, gain, timelineStartBeats, timelineEndBeats, happyFadeInBeats, happyFadeOutBeats, happyFadeInExponent, happyFadeOutExponent, gainCurve, output.getFrameCount()); for (int channel = 0; channel < outputChannelCount; channel++) { float *const channelData = output.getChannelData(channel); #pragma clang loop vectorize(enable) for (int i = 0; i < output.getFrameCount(); i++) { channelData[i] *= gainCurve[i]; } } } float AudioClip::getTransposition() const { return emscripten_atomic_load_f32(&transposition); } void AudioClip::setTransposition(float newTrans) { newTrans = std::clamp(newTrans, TRANSPOSE_MINIMUM_SEMITONES, TRANSPOSE_MAXIMUM_SEMITONES); emscripten_atomic_store_f32(&transposition, newTrans); } float AudioClip::getGain() const { return emscripten_atomic_load_f32(&gain); } void AudioClip::setGain(float newGain) { emscripten_atomic_store_f32(&gain, newGain); } double AudioClip::getLoopStartBeats() const { return emscripten_atomic_load_f64(&loopStartBeats); } void AudioClip::setLoopStartBeats(double newStart) { emscripten_atomic_store_f64(&loopStartBeats, newStart); } double AudioClip::getLoopEndBeats() const { return emscripten_atomic_load_f64(&loopEndBeats); } void AudioClip::setLoopEndBeats(double newEnd) { emscripten_atomic_store_f64(&loopEndBeats, newEnd); } double AudioClip::getReadStartBeats() const { return emscripten_atomic_load_f64(&readStartBeats); } void AudioClip::setReadStartBeats(double newStart) { emscripten_atomic_store_f64(&readStartBeats, newStart); } double AudioClip::getFadeInBeats() const { return emscripten_atomic_load_f64(&fadeInBeats); } void AudioClip::setFadeInBeats(double newFade) { emscripten_atomic_store_f64(&fadeInBeats, newFade); } double AudioClip::getFadeInExponent() const { return emscripten_atomic_load_f64(&fadeInExponent); } void AudioClip::setFadeInExponent(double newExp) { emscripten_atomic_store_f64(&fadeInExponent, newExp); } double AudioClip::getFadeOutBeats() const { return emscripten_atomic_load_f64(&fadeOutBeats); } void AudioClip::setFadeOutBeats(double newFade) { emscripten_atomic_store_f64(&fadeOutBeats, newFade); } double AudioClip::getFadeOutExponent() const { return emscripten_atomic_load_f64(&fadeOutExponent); } void AudioClip::setFadeOutExponent(double newExp) { emscripten_atomic_store_f64(&fadeOutExponent, newExp); } double AudioClip::getWarpedContentBps() const { return emscripten_atomic_load_f64(&warpedContentBps); } void AudioClip::setWarpedContentBps(double newBps) { emscripten_atomic_store_f64(&warpedContentBps, newBps); } bool AudioClip::getWarpEnabled() const { return emscripten_atomic_load_u8( reinterpret_cast(&warpEnabled)); } void AudioClip::setWarpEnabled(bool enabled) { emscripten_atomic_store_u8(reinterpret_cast(&warpEnabled), enabled); } EMSCRIPTEN_BINDINGS(audioclip) { using namespace emscripten; class_("AudioClip") .smart_ptr>("AudioClip") .property("gain", &AudioClip::getGain, &AudioClip::setGain) .property("timelineStartBeats", &AudioClip::getTimelineStartBeats) .property("timelineEndBeats", &AudioClip::getTimelineEndBeats) .property("loopStartBeats", &AudioClip::getLoopStartBeats, &AudioClip::setLoopStartBeats) .property("loopEndBeats", &AudioClip::getLoopEndBeats, &AudioClip::setLoopEndBeats) .property("readStartBeats", &AudioClip::getReadStartBeats, &AudioClip::setReadStartBeats) .property("fadeInBeats", &AudioClip::getFadeInBeats, &AudioClip::setFadeInBeats) .property("fadeInExponent", &AudioClip::getFadeInExponent, &AudioClip::setFadeInExponent) .property("fadeOutBeats", &AudioClip::getFadeOutBeats, &AudioClip::setFadeOutBeats) .property("fadeOutExponent", &AudioClip::getFadeOutExponent, &AudioClip::setFadeOutExponent) .property("transposition", &AudioClip::getTransposition, &AudioClip::setTransposition) .property("warpedContentBps", &AudioClip::getWarpedContentBps, &AudioClip::setWarpedContentBps) .property("warpEnabled", &AudioClip::getWarpEnabled, &AudioClip::setWarpEnabled); }; TEST_CASE("audioclip memory tests", "[audioclip]") { using namespace time_units; auto buf = std::make_shared(2, 44100); buf->noise(); auto out = std::make_shared(2, 128); double clock = 0.0; auto rab = std::make_shared(44100, buf); auto warpMap = std::make_shared>( std::vector::WarpMarker>{ {Beats(0.0), Seconds(0.0)}, {Beats(1.0), Seconds(1.0)}}); AudioClip clip(rab, warpMap, 44100, 1.f, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.f, 120.0, true, Uuid()); for (int i = 0; i < 10; i++) { // TODO // clip.readSegment(, *out); } }