From a9531c86a433c8b7ae1f77ff0266c27c39eca7f4 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Fri, 10 Mar 2023 11:28:33 +1100 Subject: mostly single task pipeline --- src/audio/audio_decoder.cpp | 153 ++++++++++++++++++++------------------------ 1 file changed, 68 insertions(+), 85 deletions(-) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index 9879b042..f8614478 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -2,13 +2,16 @@ #include +#include #include #include #include +#include "cbor/tinycbor/src/cborinternal_p.h" #include "freertos/FreeRTOS.h" #include "esp_heap_caps.h" +#include "esp_log.h" #include "freertos/message_buffer.h" #include "freertos/portmacro.h" @@ -21,126 +24,106 @@ namespace audio { static const char* kTag = "DEC"; -static const std::size_t kChunkSize = 1024; -static const std::size_t kReadahead = 8; - AudioDecoder::AudioDecoder() : IAudioElement(), - arena_(kChunkSize, kReadahead, MALLOC_CAP_SPIRAM), stream_info_({}), has_samples_to_send_(false), needs_more_input_(true) {} AudioDecoder::~AudioDecoder() {} -auto AudioDecoder::HasUnprocessedInput() -> bool { - return !needs_more_input_ || has_samples_to_send_; -} - -auto AudioDecoder::IsOverBuffered() -> bool { - return arena_.BlocksFree() == 0; -} - -auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> void { - stream_info_ = info; - - if (info.chunk_size) { - chunk_reader_.emplace(info.chunk_size.value()); - } else { - ESP_LOGE(kTag, "no chunk size given"); - return; +auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { + if (!std::holds_alternative(info.data)) { + return false; } + const auto& encoded = std::get(info.data); // Reuse the existing codec if we can. This will help with gapless playback, // since we can potentially just continue to decode as we were before, // without any setup overhead. + // TODO: use audio type from stream if (current_codec_ != nullptr && - current_codec_->CanHandleFile(info.path.value_or(""))) { + current_codec_->CanHandleType(encoded.type)) { current_codec_->ResetForNewStream(); - return; + return true; } - auto result = codecs::CreateCodecForFile(info.path.value_or("")); + // TODO: use audio type from stream + auto result = codecs::CreateCodecForType(encoded.type); if (result.has_value()) { current_codec_ = std::move(result.value()); } else { ESP_LOGE(kTag, "no codec for this file"); - return; - } - - stream_info_ = info; - has_sent_stream_info_ = false; -} - -auto AudioDecoder::ProcessChunk(const cpp::span& chunk) -> void { - if (current_codec_ == nullptr || !chunk_reader_) { - // Should never happen, but fail explicitly anyway. - ESP_LOGW(kTag, "received chunk without chunk size or codec"); - return; + return false; } - ESP_LOGI(kTag, "received new chunk (size %u)", chunk.size()); - current_codec_->SetInput(chunk_reader_->HandleNewData(chunk)); - needs_more_input_ = false; + return true; } -auto AudioDecoder::ProcessEndOfStream() -> void { - has_samples_to_send_ = false; - needs_more_input_ = true; - current_codec_.reset(); +auto AudioDecoder::Process(std::vector* inputs, MutableStream* output) + -> void { + // We don't really expect multiple inputs, so just pick the first that + // contains data. If none of them contain data, then we can still flush + // pending samples. + auto input = + std::find_if(inputs->begin(), inputs->end(), + [](const Stream& s) { return s.data.size_bytes() > 0; }); + + if (input != inputs->end()) { + const StreamInfo* info = input->info; + if (!stream_info_ || *stream_info_ != *info) { + // The input stream has changed! Immediately throw everything away and + // start from scratch. + // TODO: special case gapless playback? needs thought. + stream_info_ = *info; + has_samples_to_send_ = false; + has_set_stream_info_ = false; + + ProcessStreamInfo(*info); + } - SendOrBufferEvent(std::unique_ptr( - StreamEvent::CreateEndOfStream(input_events_))); -} + current_codec_->SetInput(input->data); + } -auto AudioDecoder::Process() -> void { - if (has_samples_to_send_) { - // Writing samples is relatively quick (it's just a bunch of memcopy's), so - // do them all at once. - while (has_samples_to_send_ && !IsOverBuffered()) { - if (!has_sent_stream_info_) { - has_sent_stream_info_ = true; + while (true) { + if (has_samples_to_send_) { + if (!has_set_stream_info_) { + has_set_stream_info_ = true; auto format = current_codec_->GetOutputFormat(); - stream_info_->bits_per_sample = format.bits_per_sample; - stream_info_->sample_rate = format.sample_rate_hz; - stream_info_->channels = format.num_channels; - stream_info_->chunk_size = kChunkSize; - - auto event = - StreamEvent::CreateStreamInfo(input_events_, *stream_info_); - SendOrBufferEvent(std::unique_ptr(event)); - } - - auto block = arena_.Acquire(); - if (!block) { - return; + output->info->data.emplace( + format.bits_per_sample, format.sample_rate_hz, format.num_channels); } - auto write_res = - current_codec_->WriteOutputSamples({block->start, block->size}); - block->used_size = write_res.first; + auto write_res = current_codec_->WriteOutputSamples( + output->data.subspan(output->info->bytes_in_stream)); + output->info->bytes_in_stream += write_res.first; has_samples_to_send_ = !write_res.second; - auto chunk = std::unique_ptr( - StreamEvent::CreateArenaChunk(input_events_, *block)); - if (!SendOrBufferEvent(std::move(chunk))) { - return; + if (has_samples_to_send_) { + // We weren't able to fit all the generated samples into the output + // buffer. Stop trying; we'll finish up during the next pass. + break; } } - // We will process the next frame during the next call to this method. - } - - if (!needs_more_input_) { - auto res = current_codec_->ProcessNextFrame(); - if (res.has_error()) { - // TODO(jacqueline): Handle errors. - return; - } - needs_more_input_ = res.value(); - has_samples_to_send_ = true; - if (needs_more_input_) { - chunk_reader_->HandleBytesUsed(current_codec_->GetInputPosition()); + if (input != inputs->end()) { + auto res = current_codec_->ProcessNextFrame(); + if (res.has_error()) { + // TODO(jacqueline): Handle errors. + return; + } + input->data = input->data.subspan(current_codec_->GetInputPosition()); + + if (res.value()) { + // We're out of data in this buffer. Finish immediately; there's nothing + // to send. + break; + } else { + has_samples_to_send_ = true; + } + } else { + // No input; nothing to do. + break; } } } -- cgit v1.2.3 From 7c6fd654f50e6665efa4226c6b927f9762734182 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Sat, 1 Apr 2023 13:22:21 +1100 Subject: New pipeline building, still needs proper control --- src/audio/audio_decoder.cpp | 97 +++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 48 deletions(-) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index f8614478..ada1f8f7 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -26,22 +26,22 @@ static const char* kTag = "DEC"; AudioDecoder::AudioDecoder() : IAudioElement(), - stream_info_({}), - has_samples_to_send_(false), - needs_more_input_(true) {} + current_codec_(), + current_input_format_(), + current_output_format_(), + has_samples_to_send_(false) {} AudioDecoder::~AudioDecoder() {} auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { - if (!std::holds_alternative(info.data)) { + if (!std::holds_alternative(info.format)) { return false; } - const auto& encoded = std::get(info.data); + const auto& encoded = std::get(info.format); // Reuse the existing codec if we can. This will help with gapless playback, // since we can potentially just continue to decode as we were before, // without any setup overhead. - // TODO: use audio type from stream if (current_codec_ != nullptr && current_codec_->CanHandleType(encoded.type)) { current_codec_->ResetForNewStream(); @@ -60,43 +60,47 @@ auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { return true; } -auto AudioDecoder::Process(std::vector* inputs, MutableStream* output) - -> void { +auto AudioDecoder::Process(const std::vector& inputs, + OutputStream* output) -> void { // We don't really expect multiple inputs, so just pick the first that // contains data. If none of them contain data, then we can still flush // pending samples. - auto input = - std::find_if(inputs->begin(), inputs->end(), - [](const Stream& s) { return s.data.size_bytes() > 0; }); - - if (input != inputs->end()) { - const StreamInfo* info = input->info; - if (!stream_info_ || *stream_info_ != *info) { - // The input stream has changed! Immediately throw everything away and - // start from scratch. - // TODO: special case gapless playback? needs thought. - stream_info_ = *info; - has_samples_to_send_ = false; - has_set_stream_info_ = false; - - ProcessStreamInfo(*info); - } + auto input = std::find_if( + inputs.begin(), inputs.end(), + [](const InputStream& s) { return s.data().size_bytes() > 0; }); + if (input == inputs.end()) { + input = inputs.begin(); + } + + const StreamInfo& info = input->info(); + if (!current_input_format_ || *current_input_format_ != info.format) { + // The input stream has changed! Immediately throw everything away and + // start from scratch. + current_input_format_ = info.format; + has_samples_to_send_ = false; - current_codec_->SetInput(input->data); + ProcessStreamInfo(info); } + current_codec_->SetInput(input->data()); + while (true) { if (has_samples_to_send_) { - if (!has_set_stream_info_) { - has_set_stream_info_ = true; + if (!current_output_format_) { auto format = current_codec_->GetOutputFormat(); - output->info->data.emplace( - format.bits_per_sample, format.sample_rate_hz, format.num_channels); + current_output_format_ = StreamInfo::Pcm{ + .channels = format.num_channels, + .bits_per_sample = format.bits_per_sample, + .sample_rate = format.sample_rate_hz, + }; + } + + if (!output->prepare(*current_output_format_)) { + break; } - auto write_res = current_codec_->WriteOutputSamples( - output->data.subspan(output->info->bytes_in_stream)); - output->info->bytes_in_stream += write_res.first; + auto write_res = current_codec_->WriteOutputSamples(output->data()); + output->add(write_res.first); has_samples_to_send_ = !write_res.second; if (has_samples_to_send_) { @@ -106,26 +110,23 @@ auto AudioDecoder::Process(std::vector* inputs, MutableStream* output) } } - if (input != inputs->end()) { - auto res = current_codec_->ProcessNextFrame(); - if (res.has_error()) { - // TODO(jacqueline): Handle errors. - return; - } - input->data = input->data.subspan(current_codec_->GetInputPosition()); + auto res = current_codec_->ProcessNextFrame(); + if (res.has_error()) { + // TODO(jacqueline): Handle errors. + return; + } - if (res.value()) { - // We're out of data in this buffer. Finish immediately; there's nothing - // to send. - break; - } else { - has_samples_to_send_ = true; - } - } else { - // No input; nothing to do. + if (res.value()) { + // We're out of useable data in this buffer. Finish immediately; there's + // nothing to send. + input->mark_incomplete(); break; + } else { + has_samples_to_send_ = true; } } + + input->consume(current_codec_->GetInputPosition()); } } // namespace audio -- cgit v1.2.3 From 3836768bb8b95188e6657ab69027d1d9e4b13a77 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Mon, 3 Apr 2023 14:06:30 +1000 Subject: new pipeline working(?), but the dac eludes me --- src/audio/audio_decoder.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index ada1f8f7..03c7e998 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "cbor/tinycbor/src/cborinternal_p.h" #include "freertos/FreeRTOS.h" @@ -37,6 +38,7 @@ auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { if (!std::holds_alternative(info.format)) { return false; } + ESP_LOGI(kTag, "got new stream"); const auto& encoded = std::get(info.format); // Reuse the existing codec if we can. This will help with gapless playback, @@ -45,12 +47,14 @@ auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { if (current_codec_ != nullptr && current_codec_->CanHandleType(encoded.type)) { current_codec_->ResetForNewStream(); + ESP_LOGI(kTag, "reusing existing decoder"); return true; } // TODO: use audio type from stream auto result = codecs::CreateCodecForType(encoded.type); if (result.has_value()) { + ESP_LOGI(kTag, "creating new decoder"); current_codec_ = std::move(result.value()); } else { ESP_LOGE(kTag, "no codec for this file"); @@ -73,6 +77,9 @@ auto AudioDecoder::Process(const std::vector& inputs, } const StreamInfo& info = input->info(); + if (std::holds_alternative(info.format)) { + return; + } if (!current_input_format_ || *current_input_format_ != info.format) { // The input stream has changed! Immediately throw everything away and // start from scratch. @@ -100,6 +107,9 @@ auto AudioDecoder::Process(const std::vector& inputs, } auto write_res = current_codec_->WriteOutputSamples(output->data()); + if (write_res.first > 0) { + ESP_LOGI(kTag, "wrote %u bytes of samples", write_res.first); + } output->add(write_res.first); has_samples_to_send_ = !write_res.second; -- cgit v1.2.3 From 40a9734b04c48339cfdf6ed9043aa3f6f0dda62d Mon Sep 17 00:00:00 2001 From: jacqueline Date: Tue, 4 Apr 2023 09:46:52 +1000 Subject: Redo pcm registers to include pages --- src/audio/audio_decoder.cpp | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index 03c7e998..b829f959 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -107,9 +107,6 @@ auto AudioDecoder::Process(const std::vector& inputs, } auto write_res = current_codec_->WriteOutputSamples(output->data()); - if (write_res.first > 0) { - ESP_LOGI(kTag, "wrote %u bytes of samples", write_res.first); - } output->add(write_res.first); has_samples_to_send_ = !write_res.second; -- cgit v1.2.3 From 7a54ff0df9c18b662e5bdc11ac2e26ff052cfa4d Mon Sep 17 00:00:00 2001 From: jacqueline Date: Tue, 4 Apr 2023 14:12:01 +1000 Subject: WIP track down new pipeline memory issues --- src/audio/audio_decoder.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index b829f959..faaadb3e 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -66,20 +66,13 @@ auto AudioDecoder::ProcessStreamInfo(const StreamInfo& info) -> bool { auto AudioDecoder::Process(const std::vector& inputs, OutputStream* output) -> void { - // We don't really expect multiple inputs, so just pick the first that - // contains data. If none of them contain data, then we can still flush - // pending samples. - auto input = std::find_if( - inputs.begin(), inputs.end(), - [](const InputStream& s) { return s.data().size_bytes() > 0; }); - if (input == inputs.end()) { - input = inputs.begin(); - } - + auto input = inputs.begin(); const StreamInfo& info = input->info(); - if (std::holds_alternative(info.format)) { + if (std::holds_alternative(info.format) || info.bytes_in_stream == 0) { + output->prepare({}); return; } + if (!current_input_format_ || *current_input_format_ != info.format) { // The input stream has changed! Immediately throw everything away and // start from scratch. @@ -133,7 +126,8 @@ auto AudioDecoder::Process(const std::vector& inputs, } } - input->consume(current_codec_->GetInputPosition()); + ESP_LOGI(kTag, "decoded %u bytes", current_codec_->GetInputPosition() - 1); + input->consume(current_codec_->GetInputPosition() - 1); } } // namespace audio -- cgit v1.2.3 From 561f9d2a07ee6ee1c2f18dc375125f87ea7b0d55 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Wed, 19 Apr 2023 13:00:42 +1000 Subject: Ensure the sink buffer is large enough to not fully drain during playback --- src/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index faaadb3e..af9abb94 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -68,8 +68,10 @@ auto AudioDecoder::Process(const std::vector& inputs, OutputStream* output) -> void { auto input = inputs.begin(); const StreamInfo& info = input->info(); - if (std::holds_alternative(info.format) || info.bytes_in_stream == 0) { - output->prepare({}); + if (std::holds_alternative(info.format) || + info.bytes_in_stream == 0) { + // TODO(jacqueline): should we clear the stream format? + // output->prepare({}); return; } @@ -126,7 +128,6 @@ auto AudioDecoder::Process(const std::vector& inputs, } } - ESP_LOGI(kTag, "decoded %u bytes", current_codec_->GetInputPosition() - 1); input->consume(current_codec_->GetInputPosition() - 1); } -- cgit v1.2.3 From 4c77950e702a329f3136456a932efbea36e03d42 Mon Sep 17 00:00:00 2001 From: jacqueline Date: Wed, 19 Apr 2023 16:45:50 +1000 Subject: Pipeline working and outputting correctly, but noisy --- src/audio/audio_decoder.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/audio/audio_decoder.cpp') diff --git a/src/audio/audio_decoder.cpp b/src/audio/audio_decoder.cpp index af9abb94..4b9826a9 100644 --- a/src/audio/audio_decoder.cpp +++ b/src/audio/audio_decoder.cpp @@ -128,7 +128,10 @@ auto AudioDecoder::Process(const std::vector& inputs, } } - input->consume(current_codec_->GetInputPosition() - 1); + std::size_t pos = current_codec_->GetInputPosition(); + if (pos > 0) { + input->consume(pos - 1); + } } } // namespace audio -- cgit v1.2.3