From 60169cdd91c46fb9c5cd22c2864e47b8e2dc413e Mon Sep 17 00:00:00 2001 From: jacqueline Date: Thu, 17 Nov 2022 13:30:16 +1100 Subject: WIP on our own pipeline --- src/audio/README.md | 59 +++++++++++++++++++ src/audio/audio_task.cpp | 78 +++++++++++++++++++++++++ src/audio/fatfs_audio_input.cpp | 79 +++++++++++++++++++++++++ src/audio/include/audio_backend.hpp | 20 +++++++ src/audio/include/audio_decoder.hpp | 34 +++++++++++ src/audio/include/audio_element.hpp | 73 +++++++++++++++++++++++ src/audio/include/audio_output.hpp | 29 +++++++++ src/audio/include/audio_playback.hpp | 97 +++++++++++++++++++++++++++++++ src/audio/include/audio_task.hpp | 15 +++++ src/audio/include/fatfs_audio_input.hpp | 49 ++++++++++++++++ src/codecs/CMakeLists.txt | 0 src/codecs/README.md | 3 + src/codecs/codec.hpp | 17 ++++++ src/drivers/include/audio_output.hpp | 29 --------- src/drivers/include/audio_playback.hpp | 97 ------------------------------- src/drivers/include/fatfs_audio_input.hpp | 40 +++++++++++++ 16 files changed, 593 insertions(+), 126 deletions(-) create mode 100644 src/audio/README.md create mode 100644 src/audio/audio_task.cpp create mode 100644 src/audio/fatfs_audio_input.cpp create mode 100644 src/audio/include/audio_backend.hpp create mode 100644 src/audio/include/audio_decoder.hpp create mode 100644 src/audio/include/audio_element.hpp create mode 100644 src/audio/include/audio_output.hpp create mode 100644 src/audio/include/audio_playback.hpp create mode 100644 src/audio/include/audio_task.hpp create mode 100644 src/audio/include/fatfs_audio_input.hpp create mode 100644 src/codecs/CMakeLists.txt create mode 100644 src/codecs/README.md create mode 100644 src/codecs/codec.hpp delete mode 100644 src/drivers/include/audio_output.hpp delete mode 100644 src/drivers/include/audio_playback.hpp create mode 100644 src/drivers/include/fatfs_audio_input.hpp diff --git a/src/audio/README.md b/src/audio/README.md new file mode 100644 index 00000000..e6a78a61 --- /dev/null +++ b/src/audio/README.md @@ -0,0 +1,59 @@ + +FatfsAudioReader + - input if a queue of filenames. + - output is a cbor stream + - 1 header, like "this is a new file! this is the file type! + - followed by length-prefixed chunks of bytes + - runs in a task, which prompts it to read/write one chunk, then returns. + - task watches for kill signal, owns storage, etc. + +AudioDecoder + - input is the chunked bytes above. + - output is also a cbor stream + - 1 header, which is like a reconfiguration packet thing. + - "data that follows is this depth, this sample rate" + - also indicates whether the configuration is 'sudden' for soft muting? + - then length-prefixed chunks of bytes + +AudioOutput + - input is the output of the decoder + - outputs via writing to i2s_write, which copies data to a dma buffer + - therefore, safe for us to consume any kind of reconfiguration here. + - only issue is that we will need to wait for the dma buffers to drain before + we can reconfigure the driver. (i2s_zero_dma_buffer) + - this is important for i2s speed; we should avoid extra copy steps for the raw + - pcm stream + - input therefore needs to be two channels: one configuration channel, one bytes + channel + + +How do things like seeking, and progress work? + - Reader knows where we are in terms of file size and position + - Decoder knows sample rate, frames, etc. for knowing how that maps into + - the time progress + - Output knows where we are as well in a sense, but only in terms of the PCM + output. this doesn't correspond to anything very well. + + So, to seek: + - come up with your position. this is likely "where we are plus 10", or a + specific timecode. the decoder has what we need for the byte position of this + - tell the reader "hey we need to be in this file at this byte position + - reader clears its own output buffer (since it's been doing readahead) and + starts again at the given location + For current position, the decoder will need to track where in the file it's up + to. + +HEADERS + DATA: + - cbor seems sensible for headers. allocate a little working buffer, encode the + data, then send it out on the ringbuffer. + - the data itself is harder, since tinycbor doesn't support writing chunked indefinite + length stuff. this is a problem bc we need to give cbor the buffer up front, but + we don't know exactly how long things will be, so it ends up being slightly awkward + and inefficient. + - we could also just like... write the struct i guess? that might be okay. + - gives us a format like + - could be smart with the type, use like a 32 bit int, and encode the length + - in there? + - then from the reader's perspective, it's: + - read 4 bytes, work out what's next + - read the next X bytes diff --git a/src/audio/audio_task.cpp b/src/audio/audio_task.cpp new file mode 100644 index 00000000..1853431a --- /dev/null +++ b/src/audio/audio_task.cpp @@ -0,0 +1,78 @@ +#include "audio_task.hpp" + +#include + +#include + +#include "esp_heap_caps.h" +#include "freertos/portmacro.h" +#include "freertos/queue.h" +#include "freertos/stream_buffer.h" + +#include "audio_element.hpp" + +namespace audio { + +static const TickType_t kCommandWaitTicks = 1; + +void audio_task(void* args) { + AudioTaskArgs* real_args = reinterpret_cast(args); + std::shared_ptr element = real_args->element; + delete real_args; + + QueueHandle_t commands = element->InputCommandQueue(); + StreamBufferHandle_t stream = element->InputBuffer(); + + // TODO: think about overflow. + uint8_t current_sequence_number; + uint8_t* frame_buffer = + (uint8_t*)heap_caps_malloc(kFrameSize, MALLOC_CAP_SPIRAM); + + while (1) { + IAudioElement::Command command; + if (!xQueueReceive(commands, &command, kCommandWaitTicks)) { + element->ProcessIdle(); + continue; + }; + + if (command.type == IAudioElement::SEQUENCE_NUMBER) { + if (command.sequence_number > current_sequence_number) { + current_sequence_number = command.sequence_number; + } + + continue; + } + + if (command.type == IAudioElement::READ) { + assert(command.read_size <= kFrameSize); + assert(stream != NULL); + xStreamBufferReceive(stream, &frame_buffer, command.read_size, 0); + + if (command.sequence_number == current_sequence_number) { + element->ProcessData(frame_buffer, command.read_size); + } + + continue; + } + + if (command.type == IAudioElement::ELEMENT) { + assert(command.data != NULL); + if (command.sequence_number == current_sequence_number) { + element->ProcessElementCommand(command.data); + } else { + element->SkipElementCommand(command.data); + } + } + + if (command.type == IAudioElement::QUIT) { + break; + } + } + + element = nullptr; + free(frame_buffer); + + xTaskDelete(NULL); +} + +} // namespace audio diff --git a/src/audio/fatfs_audio_input.cpp b/src/audio/fatfs_audio_input.cpp new file mode 100644 index 00000000..1e8c35b8 --- /dev/null +++ b/src/audio/fatfs_audio_input.cpp @@ -0,0 +1,79 @@ +#include "fatfs_audio_input.hpp" +#include + +#include "esp-adf/components/input_key_service/include/input_key_service.h" +#include "esp_heap_caps.h" + +#include "audio_element.hpp" + +namespace audio { + +static const size_t kQueueItems = 0; +static constexpr size_t kQueueItemSize = sizeof(IAudioElement::Command); +static constexpr size_t kQueueSize = kQueueItems * kQueueItemSize; + +static const size_t kOutputBufferSize = 1024; + +FatfsAudioInput::FatfsAudioInput(std::shared_ptr storage) + : IAudioElement(), storage_(storage) { + input_queue_memory_ = heap_caps_malloc(kQueueSize, MALLOC_CAP_SPIRAM); + input_queue_ = xQueueCreateStatic( + kQueueItems, kQueueItemSize, input_queue_memory_, &input_queue_metadata_); + + output_queue_memory_ = heap_caps_malloc(kQueueSize, MALLOC_CAP_SPIRAM); + output_queue_ = + xQueueCreateStatic(kQueueItems, kQueueItemSize, output_queue_memory_, + &output_queue_metadata_); + + output_buffer_memory_ = + heap_caps_malloc(kOutputBufferSize, MALLOC_CAP_SPIRAM); + output_buffer_ = + xStreamBufferCreateStatic(kOutputBufferSize - 1, 1, output_buffer_memory_, + &output_buffer_metadata_); +} + +FatfsAudioInput::~FatfsAudioInput() { + vStreamBufferDelete(output_buffer_); + free(output_buffer_memory_); + vQueueDelete(output_queue_); + free(output_queue_memory_); + vQueueDelete(input_queue_); + free(input_queue_memory_); +} + +auto FatfsAudioInput::InputCommandQueue() -> QueueHandle_t { + return input_queue_; +} + +auto FatfsAudioInput::OutputCommandQueue() -> QueueHandle_t { + return output_queue_; +} + +auto FatfsAudioInput::InputBuffer() -> StreamBufferHandle_t { + return nullptr; +} + +auto FatfsAudioInput::OutputBuffer() -> StreamBufferHandle_t { + return output_buffer_; +} + +auto FatfsAudioInput::ProcessElementCommand(void* command) -> void { + InputCommand *real = std::reinterpret_pointer_cast(command); + + // TODO. +} + +auto FatfsAudioInput::SkipElementCommand(void* command) -> void { + InputCommand *real = std::reinterpret_pointer_cast(command); + delete real; +} + +auto FatfsAudioInput::ProcessData(uint8_t* data, uint16_t length) -> void { + // Not implemented. +} + +auto FatfsAudioInput::ProcessIdle() -> void { + // TODO. +} + +} // namespace audio diff --git a/src/audio/include/audio_backend.hpp b/src/audio/include/audio_backend.hpp new file mode 100644 index 00000000..85985cc2 --- /dev/null +++ b/src/audio/include/audio_backend.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include +namespace drivers { + +class IAudioBackend { + public: + virtual ~IAudioBackend() {} + + enum SampleRate {}; + enum BitDepth {}; + + virtual auto Configure(SampleRate sample_rate, BitDepth bit_depth) + -> bool = 0; + virtual auto WritePcmData(uint8_t* data, size_t length) -> bool = 0; + + virtual auto SetVolume(uint8_t percent) -> void = 0; +}; + +} // namespace drivers diff --git a/src/audio/include/audio_decoder.hpp b/src/audio/include/audio_decoder.hpp new file mode 100644 index 00000000..f460f9e9 --- /dev/null +++ b/src/audio/include/audio_decoder.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include "ff.h" + +namespace audio { + +enum SampleRate {}; +enum BitDepth {}; + +struct PcmStreamHeader { + SampleRate sample_rate; + BitDepth bit_depth; + bool configure_now; +}; + +class AudioDecoder { + public: + AudioDecoder(); + ~AudioDecoder(); + + auto SetSource(RingbufHandle_t& source) -> void; + + enum Status {}; + auto ProcessChunk() -> Status; + + auto GetOutputStream() const -> RingbufHandle_t; + + private: + RingbufHandle_t input_; + RingbufHandle_t output_; +}; + +} // namespace audio diff --git a/src/audio/include/audio_element.hpp b/src/audio/include/audio_element.hpp new file mode 100644 index 00000000..ea4256ac --- /dev/null +++ b/src/audio/include/audio_element.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include + +namespace audio { + +extern const std::size_t kMaxFrameSize; + +class IAudioElement { + public: + virtual ~IAudioElement(); + + enum CommandType { + /* + * Sets the sequence number of the most recent byte stream. Any commands + * received that have a lower sequence number than this will be discarded. + */ + SEQUENCE_NUMBER, + /* + * Instructs this element to read a specific number of bytes from its + * input buffer. + */ + READ_FRAME, + /* + * Represents an element-specific command. This handling of this is + * delegated to element implementations. + */ + ELEMENT, + /* Instructs this element to shut down. */ + QUIT, + }; + + struct Command { + CommandType type; + uint8_t sequence_number; + union { + void* data; + std::size_t frame_size; + }; + }; + + /* + * Returns a queue that should be used for all communication with this + * element. + */ + virtual auto InputCommandQueue() -> QueueHandle_t = 0; + + /* + * Returns a buffer that will be used to stream input bytes to this element. + * This may be NULL, if this element represents a source, e.g. a FATFS + * reader. + */ + virtual auto InputBuffer() -> StreamBufferHandle_t = 0; + + /* + * Called when an element-specific command has been received. + */ + virtual auto ProcessElementCommand(void* command) -> void = 0; + + virtual auto SkipElementCommand(void* command) -> void = 0; + + /* + * Called with the result of a read bytes command. + */ + virtual auto ProcessData(uint8_t* data, uint16_t length) -> void = 0; + + /* + * Called periodically when there are no pending commands. + */ + virtual auto ProcessIdle() -> void = 0; +}; + +} // namespace audio diff --git a/src/audio/include/audio_output.hpp b/src/audio/include/audio_output.hpp new file mode 100644 index 00000000..82dca82d --- /dev/null +++ b/src/audio/include/audio_output.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#include "audio_common.h" +#include "audio_element.h" + +namespace drivers { + +class IAudioOutput { + public: + IAudioOutput(audio_element_handle_t element) : element_(element) {} + virtual ~IAudioOutput() { audio_element_deinit(element_); } + + auto GetAudioElement() -> audio_element_handle_t { return element_; } + + virtual auto SetVolume(uint8_t volume) -> void = 0; + virtual auto GetVolume() const -> uint8_t { return volume_; } + + virtual auto Configure(audio_element_info_t& info) -> void = 0; + virtual auto SetSoftMute(bool enabled) -> void = 0; + + protected: + audio_element_handle_t element_; + uint8_t volume_; +}; + +} // namespace drivers diff --git a/src/audio/include/audio_playback.hpp b/src/audio/include/audio_playback.hpp new file mode 100644 index 00000000..41ab46d2 --- /dev/null +++ b/src/audio/include/audio_playback.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include + +#include "audio_common.h" +#include "audio_element.h" +#include "audio_event_iface.h" +#include "audio_pipeline.h" +#include "esp_err.h" +#include "fatfs_stream.h" +#include "i2s_stream.h" +#include "mp3_decoder.h" +#include "result.hpp" + +#include "audio_output.hpp" +#include "dac.hpp" +#include "storage.hpp" + +namespace drivers { + +/* + * Sends an I2S audio stream to the DAC. Includes basic controls for pausing + * and resuming the stream, as well as support for gapless playback of the next + * queued song, but does not implement any kind of sophisticated queing or + * playback control; these should be handled at a higher level. + */ +class AudioPlayback { + public: + enum Error { FATFS_INIT, I2S_INIT, PIPELINE_INIT }; + static auto create(std::unique_ptr output) + -> cpp::result, Error>; + + AudioPlayback(std::unique_ptr& output, + audio_pipeline_handle_t pipeline, + audio_element_handle_t source_element, + audio_event_iface_handle_t event_interface); + ~AudioPlayback(); + + /* + * Replaces any currently playing file with the one given, and begins + * playback. + * + * Any value set in `set_next_file` is cleared by this method. + */ + auto Play(const std::string& filename) -> void; + /* Toogle between resumed and paused. */ + auto Toggle() -> void; + auto Resume() -> void; + auto Pause() -> void; + + enum PlaybackState { PLAYING, PAUSED, STOPPED }; + auto GetPlaybackState() const -> PlaybackState; + + /* + * Handles any pending events from the underlying audio pipeline. This must + * be called regularly in order to handle configuring the I2S stream for + * different audio types (e.g. sample rate, bit depth), and for gapless + * playback. + */ + auto ProcessEvents(uint16_t max_time_ms) -> void; + + /* + * Sets the file that should be played immediately after the current file + * finishes. This is used for gapless playback + */ + auto SetNextFile(const std::string& filename) -> void; + + auto SetVolume(uint8_t volume) -> void; + auto GetVolume() const -> uint8_t; + + // Not copyable or movable. + AudioPlayback(const AudioPlayback&) = delete; + AudioPlayback& operator=(const AudioPlayback&) = delete; + + private: + PlaybackState playback_state_; + + enum Decoder { NONE, MP3, AMR, OPUS, OGG, FLAC, WAV, AAC }; + auto GetDecoderForFilename(std::string filename) const -> Decoder; + auto CreateDecoder(Decoder decoder) const -> audio_element_handle_t; + auto ReconfigurePipeline(Decoder decoder) -> void; + + std::unique_ptr output_; + + std::string next_filename_ = ""; + + audio_pipeline_handle_t pipeline_; + audio_element_handle_t source_element_; + audio_event_iface_handle_t event_interface_; + + audio_element_handle_t decoder_ = nullptr; + Decoder decoder_type_ = NONE; +}; + +} // namespace drivers diff --git a/src/audio/include/audio_task.hpp b/src/audio/include/audio_task.hpp new file mode 100644 index 00000000..79604f33 --- /dev/null +++ b/src/audio/include/audio_task.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include "audio_element.hpp" + +namespace audio { + +struct AudioTaskArgs { + std::shared_ptr& element; +}; + +void audio_task(void* args); + +} // namespace audio diff --git a/src/audio/include/fatfs_audio_input.hpp b/src/audio/include/fatfs_audio_input.hpp new file mode 100644 index 00000000..ed4da55e --- /dev/null +++ b/src/audio/include/fatfs_audio_input.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/stream_buffer.h" + +#include "audio_element.hpp" +#include "storage.hpp" + +namespace audio { + +class FatfsAudioInput : public IAudioElement { + public: + struct InputCommand { + std::string filename; + }; + + struct OutputCommand { + // TODO: does this actually need any special output? + }; + + FatfsAudioInput(std::shared_ptr storage); + ~FatfsAudioInput(); + + auto OutputCommandQueue() -> QueueHandle_t; + auto OutputBuffer() -> StreamBufferHandle_t; + + private: + std::shared_ptr storage_; + + uint8_t current_sequence = 0; + + uint8_t* input_queue_memory_; + StaticQueue_t input_queue_metadata_; + QueueHandle_t input_queue_; + + uint8_t* output_queue_memory_; + StaticQueue_t output_queue_metadata_; + QueueHandle_t output_queue_; + + uint8_t* output_buffer_memory_; + StaticStreamBuffer_t output_buffer_metadata_; + StreamBufferHandle_t output_buffer_; +}; + +} // namespace audio diff --git a/src/codecs/CMakeLists.txt b/src/codecs/CMakeLists.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/codecs/README.md b/src/codecs/README.md new file mode 100644 index 00000000..06e0bfde --- /dev/null +++ b/src/codecs/README.md @@ -0,0 +1,3 @@ +# Software Codecs + +This component contains a collection of software decoders for various diff --git a/src/codecs/codec.hpp b/src/codecs/codec.hpp new file mode 100644 index 00000000..24ba9cfe --- /dev/null +++ b/src/codecs/codec.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +namespace codecs { + + class IAudioDecoder { + public: + virtual ~IAudioDecoder() {} + + virtual auto ProcessData( + uint8_t *input, + size_t input_len, + uint8_t *output) -> size_t = 0; + }; + +} // namespace codecs diff --git a/src/drivers/include/audio_output.hpp b/src/drivers/include/audio_output.hpp deleted file mode 100644 index 82dca82d..00000000 --- a/src/drivers/include/audio_output.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include - -#include "audio_common.h" -#include "audio_element.h" - -namespace drivers { - -class IAudioOutput { - public: - IAudioOutput(audio_element_handle_t element) : element_(element) {} - virtual ~IAudioOutput() { audio_element_deinit(element_); } - - auto GetAudioElement() -> audio_element_handle_t { return element_; } - - virtual auto SetVolume(uint8_t volume) -> void = 0; - virtual auto GetVolume() const -> uint8_t { return volume_; } - - virtual auto Configure(audio_element_info_t& info) -> void = 0; - virtual auto SetSoftMute(bool enabled) -> void = 0; - - protected: - audio_element_handle_t element_; - uint8_t volume_; -}; - -} // namespace drivers diff --git a/src/drivers/include/audio_playback.hpp b/src/drivers/include/audio_playback.hpp deleted file mode 100644 index 41ab46d2..00000000 --- a/src/drivers/include/audio_playback.hpp +++ /dev/null @@ -1,97 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "audio_common.h" -#include "audio_element.h" -#include "audio_event_iface.h" -#include "audio_pipeline.h" -#include "esp_err.h" -#include "fatfs_stream.h" -#include "i2s_stream.h" -#include "mp3_decoder.h" -#include "result.hpp" - -#include "audio_output.hpp" -#include "dac.hpp" -#include "storage.hpp" - -namespace drivers { - -/* - * Sends an I2S audio stream to the DAC. Includes basic controls for pausing - * and resuming the stream, as well as support for gapless playback of the next - * queued song, but does not implement any kind of sophisticated queing or - * playback control; these should be handled at a higher level. - */ -class AudioPlayback { - public: - enum Error { FATFS_INIT, I2S_INIT, PIPELINE_INIT }; - static auto create(std::unique_ptr output) - -> cpp::result, Error>; - - AudioPlayback(std::unique_ptr& output, - audio_pipeline_handle_t pipeline, - audio_element_handle_t source_element, - audio_event_iface_handle_t event_interface); - ~AudioPlayback(); - - /* - * Replaces any currently playing file with the one given, and begins - * playback. - * - * Any value set in `set_next_file` is cleared by this method. - */ - auto Play(const std::string& filename) -> void; - /* Toogle between resumed and paused. */ - auto Toggle() -> void; - auto Resume() -> void; - auto Pause() -> void; - - enum PlaybackState { PLAYING, PAUSED, STOPPED }; - auto GetPlaybackState() const -> PlaybackState; - - /* - * Handles any pending events from the underlying audio pipeline. This must - * be called regularly in order to handle configuring the I2S stream for - * different audio types (e.g. sample rate, bit depth), and for gapless - * playback. - */ - auto ProcessEvents(uint16_t max_time_ms) -> void; - - /* - * Sets the file that should be played immediately after the current file - * finishes. This is used for gapless playback - */ - auto SetNextFile(const std::string& filename) -> void; - - auto SetVolume(uint8_t volume) -> void; - auto GetVolume() const -> uint8_t; - - // Not copyable or movable. - AudioPlayback(const AudioPlayback&) = delete; - AudioPlayback& operator=(const AudioPlayback&) = delete; - - private: - PlaybackState playback_state_; - - enum Decoder { NONE, MP3, AMR, OPUS, OGG, FLAC, WAV, AAC }; - auto GetDecoderForFilename(std::string filename) const -> Decoder; - auto CreateDecoder(Decoder decoder) const -> audio_element_handle_t; - auto ReconfigurePipeline(Decoder decoder) -> void; - - std::unique_ptr output_; - - std::string next_filename_ = ""; - - audio_pipeline_handle_t pipeline_; - audio_element_handle_t source_element_; - audio_event_iface_handle_t event_interface_; - - audio_element_handle_t decoder_ = nullptr; - Decoder decoder_type_ = NONE; -}; - -} // namespace drivers diff --git a/src/drivers/include/fatfs_audio_input.hpp b/src/drivers/include/fatfs_audio_input.hpp new file mode 100644 index 00000000..3753c136 --- /dev/null +++ b/src/drivers/include/fatfs_audio_input.hpp @@ -0,0 +1,40 @@ +#pragma once + +namespace drivers { + + class FatfsAudioInput { + public: + FatfsAudioInput(std::shared_ptr storage); + ~FatfsAudioInput(); + + enum Status { + /* + * Successfully read data into the output buffer, and there is still + * data remaining in the file. + */ + OKAY, + + /* + * The ringbuffer was full. No data was read. + */ + RINGBUF_FULL, + + /* + * Some data may have been read into the output buffer, but the file is + * now empty. + */ + FILE_EMPTY, + }; + auto Process() -> Status; + + auto GetOutputBuffer() -> RingbufHandle_t; + + private: + std::shared_ptr storage_; + RingbufHandle_t output_; + + std::string path_; + + }; + +} // namespace drivers -- cgit v1.2.3