#include "speaker_media_player.h" #ifdef USE_ESP32 #include "esphome/core/log.h" #include "esphome/components/audio/audio.h" #ifdef USE_OTA #include "esphome/components/ota/ota_backend.h" #endif namespace esphome { namespace speaker { // Framework: // - Media player that can handle two streams: one for media and one for announcements // - Each stream has an individual speaker component for output // - Each stream is handled by an ``AudioPipeline`` object with two parts/tasks // - ``AudioReader`` handles reading from an HTTP source or from a PROGMEM flash set at compile time // - ``AudioDecoder`` handles decoding the audio file. All formats are limited to two channels and 16 bits per sample // - FLAC // - MP3 (based on the libhelix decoder) // - WAV // - Each task runs until it is done processing the file or it receives a stop command // - Inter-task communication uses a FreeRTOS Event Group // - The ``AudioPipeline`` sets up a ring buffer between the reader and decoder tasks. The decoder task outputs audio // directly to a speaker component. // - The pipelines internal state needs to be processed by regularly calling ``process_state``. // - Generic media player commands are received by the ``control`` function. The commands are added to the // ``media_control_command_queue_`` to be processed in the component's loop // - Local file play back is initiatied with ``play_file`` and adds it to the ``media_control_command_queue_`` // - Starting a stream intializes the appropriate pipeline or stops it if it is already running // - Volume and mute commands are achieved by the ``mute``, ``unmute``, ``set_volume`` functions. // - Volume commands are ignored if the media control queue is full to avoid crashing with rapid volume // increases/decreases. // - These functions all send the appropriate information to the speakers to implement. // - Pausing is implemented in the decoder task and is also sent directly to the media speaker component to decrease // latency. // - The components main loop performs housekeeping: // - It reads the media control queue and processes it directly // - It determines the overall state of the media player by considering the state of each pipeline // - announcement playback takes highest priority // - Handles playlists and repeating by starting the appropriate file when a previous file is finished // - Logging only happens in the main loop task to reduce task stack memory usage. static const uint32_t MEDIA_CONTROLS_QUEUE_LENGTH = 20; static const UBaseType_t MEDIA_PIPELINE_TASK_PRIORITY = 1; static const UBaseType_t ANNOUNCEMENT_PIPELINE_TASK_PRIORITY = 1; static const char *const TAG = "speaker_media_player"; void SpeakerMediaPlayer::setup() { #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF state = media_player::MEDIA_PLAYER_STATE_OFF; #else state = media_player::MEDIA_PLAYER_STATE_IDLE; #endif this->media_control_command_queue_ = xQueueCreate(MEDIA_CONTROLS_QUEUE_LENGTH, sizeof(MediaCallCommand)); this->pref_ = this->make_entity_preference(); VolumeRestoreState volume_restore_state; if (this->pref_.load(&volume_restore_state)) { this->set_volume_(volume_restore_state.volume); this->set_mute_state_(volume_restore_state.is_muted); } else { this->set_volume_(this->volume_initial_); this->set_mute_state_(false); } #ifdef USE_OTA_STATE_LISTENER ota::get_global_ota_callback()->add_global_state_listener(this); #endif this->announcement_pipeline_ = make_unique(this->announcement_speaker_, this->buffer_size_, this->task_stack_in_psram_, "ann", ANNOUNCEMENT_PIPELINE_TASK_PRIORITY); if (this->announcement_pipeline_ == nullptr) { ESP_LOGE(TAG, "Failed to create announcement pipeline"); this->mark_failed(); } if (!this->single_pipeline_()) { this->media_pipeline_ = make_unique(this->media_speaker_, this->buffer_size_, this->task_stack_in_psram_, "med", MEDIA_PIPELINE_TASK_PRIORITY); if (this->media_pipeline_ == nullptr) { ESP_LOGE(TAG, "Failed to create media pipeline"); this->mark_failed(); } } ESP_LOGI(TAG, "Set up speaker media player"); } void SpeakerMediaPlayer::set_playlist_delay_ms(AudioPipelineType pipeline_type, uint32_t delay_ms) { switch (pipeline_type) { case AudioPipelineType::ANNOUNCEMENT: this->announcement_playlist_delay_ms_ = delay_ms; break; case AudioPipelineType::MEDIA: this->media_playlist_delay_ms_ = delay_ms; break; } } void SpeakerMediaPlayer::stop_and_unpause_media_() { this->media_pipeline_->stop(); this->unpause_media_remaining_ = 3; this->set_interval("unpause_med", 50, [this]() { if (this->media_pipeline_state_ == AudioPipelineState::STOPPED) { this->cancel_interval("unpause_med"); this->media_pipeline_->set_pause_state(false); this->is_paused_ = false; } else if (--this->unpause_media_remaining_ == 0) { this->cancel_interval("unpause_med"); } }); } void SpeakerMediaPlayer::watch_media_commands_() { if (!this->is_ready()) { return; } MediaCallCommand media_command; if (xQueueReceive(this->media_control_command_queue_, &media_command, 0) == pdTRUE) { bool enqueue = media_command.enqueue.has_value() && media_command.enqueue.value(); if (media_command.url.has_value() || media_command.file.has_value()) { #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->state == media_player::MEDIA_PLAYER_STATE_OFF) { this->state = media_player::MEDIA_PLAYER_STATE_ON; publish_state(); } #endif PlaylistItem playlist_item; if (media_command.url.has_value()) { playlist_item.url = *media_command.url.value(); delete media_command.url.value(); } if (media_command.file.has_value()) { playlist_item.file = media_command.file; } if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) { if (!enqueue) { // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause this->cancel_timeout("next_ann"); this->announcement_playlist_.clear(); if (media_command.file.has_value()) { this->announcement_pipeline_->start_file(playlist_item.file.value()); } else if (media_command.url.has_value()) { this->announcement_pipeline_->start_url(playlist_item.url.value()); } this->announcement_pipeline_->set_pause_state(false); } this->announcement_playlist_.push_back(playlist_item); } else { if (!enqueue) { // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause this->cancel_timeout("next_media"); this->media_playlist_.clear(); if (this->is_paused_) { // If paused, stop the media pipeline and unpause it after confirming its stopped. This avoids playing a // short segment of the paused file before starting the new one. this->stop_and_unpause_media_(); } else { // Not paused, just directly start the file if (media_command.file.has_value()) { this->media_pipeline_->start_file(playlist_item.file.value()); } else if (media_command.url.has_value()) { this->media_pipeline_->start_url(playlist_item.url.value()); } this->media_pipeline_->set_pause_state(false); this->is_paused_ = false; } } this->media_playlist_.push_back(playlist_item); } return; // Don't process the new file play command further } if (media_command.volume.has_value()) { this->set_volume_(media_command.volume.value()); this->publish_state(); } if (media_command.command.has_value()) { switch (media_command.command.value()) { case media_player::MEDIA_PLAYER_COMMAND_PLAY: #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->state == media_player::MEDIA_PLAYER_STATE_OFF) { this->state = media_player::MEDIA_PLAYER_STATE_ON; publish_state(); } #endif if ((this->media_pipeline_ != nullptr) && (this->is_paused_)) { this->media_pipeline_->set_pause_state(false); } this->is_paused_ = false; break; case media_player::MEDIA_PLAYER_COMMAND_PAUSE: if ((this->media_pipeline_ != nullptr) && (!this->is_paused_)) { this->media_pipeline_->set_pause_state(true); } this->is_paused_ = true; break; #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF case media_player::MEDIA_PLAYER_COMMAND_TURN_ON: if (this->state == media_player::MEDIA_PLAYER_STATE_OFF) { this->state = media_player::MEDIA_PLAYER_STATE_ON; this->publish_state(); } break; case media_player::MEDIA_PLAYER_COMMAND_TURN_OFF: this->is_turn_off_ = true; // Intentional Fall-through #endif case media_player::MEDIA_PLAYER_COMMAND_STOP: // Pipelines do not stop immediately after calling the stop command, so confirm its stopped before unpausing. // This avoids an audible short segment playing after receiving the stop command in a paused state. #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value()) || (this->is_turn_off_ && this->announcement_pipeline_state_ != AudioPipelineState::STOPPED)) { #else if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) { #endif if (this->announcement_pipeline_ != nullptr) { this->cancel_timeout("next_ann"); this->announcement_playlist_.clear(); this->announcement_pipeline_->stop(); this->unpause_announcement_remaining_ = 3; this->set_interval("unpause_ann", 50, [this]() { if (this->announcement_pipeline_state_ == AudioPipelineState::STOPPED) { this->cancel_interval("unpause_ann"); this->announcement_pipeline_->set_pause_state(false); } else if (--this->unpause_announcement_remaining_ == 0) { this->cancel_interval("unpause_ann"); } }); } } else { if (this->media_pipeline_ != nullptr) { this->cancel_timeout("next_media"); this->media_playlist_.clear(); this->stop_and_unpause_media_(); } } break; case media_player::MEDIA_PLAYER_COMMAND_TOGGLE: if (this->media_pipeline_ != nullptr) { if (this->is_paused_) { this->media_pipeline_->set_pause_state(false); this->is_paused_ = false; } else { this->media_pipeline_->set_pause_state(true); this->is_paused_ = true; } } break; case media_player::MEDIA_PLAYER_COMMAND_MUTE: { this->set_mute_state_(true); this->publish_state(); break; } case media_player::MEDIA_PLAYER_COMMAND_UNMUTE: this->set_mute_state_(false); this->publish_state(); break; case media_player::MEDIA_PLAYER_COMMAND_VOLUME_UP: this->set_volume_(std::min(1.0f, this->volume + this->volume_increment_)); this->publish_state(); break; case media_player::MEDIA_PLAYER_COMMAND_VOLUME_DOWN: this->set_volume_(std::max(0.0f, this->volume - this->volume_increment_)); this->publish_state(); break; case media_player::MEDIA_PLAYER_COMMAND_REPEAT_ONE: if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) { this->announcement_repeat_one_ = true; } else { this->media_repeat_one_ = true; } break; case media_player::MEDIA_PLAYER_COMMAND_REPEAT_OFF: if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) { this->announcement_repeat_one_ = false; } else { this->media_repeat_one_ = false; } break; case media_player::MEDIA_PLAYER_COMMAND_CLEAR_PLAYLIST: if (this->single_pipeline_() || (media_command.announce.has_value() && media_command.announce.value())) { if (this->announcement_playlist_.empty()) { this->announcement_playlist_.resize(1); } } else { if (this->media_playlist_.empty()) { this->media_playlist_.resize(1); } } break; default: break; } } } } #ifdef USE_OTA_STATE_LISTENER void SpeakerMediaPlayer::on_ota_global_state(ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) { if (state == ota::OTA_STARTED) { if (this->media_pipeline_ != nullptr) { this->media_pipeline_->suspend_tasks(); } if (this->announcement_pipeline_ != nullptr) { this->announcement_pipeline_->suspend_tasks(); } } else if (state == ota::OTA_ERROR) { if (this->media_pipeline_ != nullptr) { this->media_pipeline_->resume_tasks(); } if (this->announcement_pipeline_ != nullptr) { this->announcement_pipeline_->resume_tasks(); } } } #endif void SpeakerMediaPlayer::loop() { this->watch_media_commands_(); // Determine state of the media player media_player::MediaPlayerState old_state = this->state; AudioPipelineState old_media_pipeline_state = this->media_pipeline_state_; if (this->media_pipeline_ != nullptr) { this->media_pipeline_state_ = this->media_pipeline_->process_state(); } if (this->media_pipeline_state_ == AudioPipelineState::ERROR_READING) { ESP_LOGE(TAG, "The media pipeline's file reader encountered an error."); } else if (this->media_pipeline_state_ == AudioPipelineState::ERROR_DECODING) { ESP_LOGE(TAG, "The media pipeline's audio decoder encountered an error."); } AudioPipelineState old_announcement_pipeline_state = this->announcement_pipeline_state_; if (this->announcement_pipeline_ != nullptr) { this->announcement_pipeline_state_ = this->announcement_pipeline_->process_state(); } if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_READING) { ESP_LOGE(TAG, "The announcement pipeline's file reader encountered an error."); } else if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_DECODING) { ESP_LOGE(TAG, "The announcement pipeline's audio decoder encountered an error."); } if (this->announcement_pipeline_state_ != AudioPipelineState::STOPPED) { this->state = media_player::MEDIA_PLAYER_STATE_ANNOUNCING; } else { if (!this->announcement_playlist_.empty()) { uint32_t timeout_ms = 0; if (old_announcement_pipeline_state == AudioPipelineState::PLAYING) { // Finished the current announcement file if (!this->announcement_repeat_one_) { // Pop item off the playlist if repeat is disabled this->announcement_playlist_.pop_front(); } // Only delay starting playback if moving on the next playlist item or repeating the current item timeout_ms = this->announcement_playlist_delay_ms_; } if (!this->announcement_playlist_.empty()) { // Start the next announcement file PlaylistItem playlist_item = this->announcement_playlist_.front(); if (playlist_item.url.has_value()) { this->announcement_pipeline_->start_url(playlist_item.url.value()); } else if (playlist_item.file.has_value()) { this->announcement_pipeline_->start_file(playlist_item.file.value()); } if (timeout_ms > 0) { // Pause pipeline internally to facilitate the delay between items this->announcement_pipeline_->set_pause_state(true); // Internally unpause the pipeline after the delay between playlist items. Announcements do not follow the // media player's pause state. this->set_timeout("next_ann", timeout_ms, [this]() { this->announcement_pipeline_->set_pause_state(false); }); } } } else { if (this->is_paused_) { #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) { this->state = media_player::MEDIA_PLAYER_STATE_PAUSED; } #else this->state = media_player::MEDIA_PLAYER_STATE_PAUSED; #endif } else if (this->media_pipeline_state_ == AudioPipelineState::PLAYING) { this->state = media_player::MEDIA_PLAYER_STATE_PLAYING; } else if (this->media_pipeline_state_ == AudioPipelineState::STOPPED) { if (!media_playlist_.empty()) { uint32_t timeout_ms = 0; if (old_media_pipeline_state == AudioPipelineState::PLAYING) { // Finished the current media file if (!this->media_repeat_one_) { // Pop item off the playlist if repeat is disabled this->media_playlist_.pop_front(); } // Only delay starting playback if moving on the next playlist item or repeating the current item timeout_ms = this->media_playlist_delay_ms_; } if (!this->media_playlist_.empty()) { PlaylistItem playlist_item = this->media_playlist_.front(); if (playlist_item.url.has_value()) { this->media_pipeline_->start_url(playlist_item.url.value()); } else if (playlist_item.file.has_value()) { this->media_pipeline_->start_file(playlist_item.file.value()); } if (timeout_ms > 0) { // Pause pipeline internally to facilitate the delay between items this->media_pipeline_->set_pause_state(true); // Internally unpause the pipeline after the delay between playlist items, if the media player state is // not paused. this->set_timeout("next_media", timeout_ms, [this]() { this->media_pipeline_->set_pause_state(this->is_paused_); }); } } } else { #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) { this->state = media_player::MEDIA_PLAYER_STATE_IDLE; } #else this->state = media_player::MEDIA_PLAYER_STATE_IDLE; #endif } } } } if (this->state != old_state) { this->publish_state(); ESP_LOGD(TAG, "State changed to %s", media_player::media_player_state_to_string(this->state)); } #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->is_turn_off_ && (this->state == media_player::MEDIA_PLAYER_STATE_PAUSED || this->state == media_player::MEDIA_PLAYER_STATE_IDLE)) { this->is_turn_off_ = false; if (this->state == media_player::MEDIA_PLAYER_STATE_PAUSED) { this->state = media_player::MEDIA_PLAYER_STATE_IDLE; this->publish_state(); ESP_LOGD(TAG, "State changed to %s", media_player::media_player_state_to_string(this->state)); } this->state = media_player::MEDIA_PLAYER_STATE_OFF; this->publish_state(); ESP_LOGD(TAG, "State changed to %s", media_player::media_player_state_to_string(this->state)); } #endif } void SpeakerMediaPlayer::play_file(audio::AudioFile *media_file, bool announcement, bool enqueue) { if (!this->is_ready()) { // Ignore any commands sent before the media player is setup return; } MediaCallCommand media_command; media_command.file = media_file; if (this->single_pipeline_() || announcement) { media_command.announce = true; } else { media_command.announce = false; } media_command.enqueue = enqueue; xQueueSend(this->media_control_command_queue_, &media_command, portMAX_DELAY); } void SpeakerMediaPlayer::control(const media_player::MediaPlayerCall &call) { if (!this->is_ready()) { // Ignore any commands sent before the media player is setup return; } MediaCallCommand media_command; auto ann = call.get_announcement(); if (this->single_pipeline_() || (ann.has_value() && *ann)) { media_command.announce = true; } else { media_command.announce = false; } auto media_url = call.get_media_url(); if (media_url.has_value()) { media_command.url = new std::string(*media_url); // Must be manually deleted after receiving media_command from a queue auto cmd = call.get_command(); if (cmd.has_value()) { if (*cmd == media_player::MEDIA_PLAYER_COMMAND_ENQUEUE) { media_command.enqueue = true; } } xQueueSend(this->media_control_command_queue_, &media_command, portMAX_DELAY); return; } auto vol = call.get_volume(); if (vol.has_value()) { media_command.volume = vol; // Wait 0 ticks for queue to be free, volume sets aren't that important! xQueueSend(this->media_control_command_queue_, &media_command, 0); return; } auto cmd = call.get_command(); if (cmd.has_value()) { media_command.command = cmd; TickType_t ticks_to_wait = portMAX_DELAY; if ((*cmd == media_player::MEDIA_PLAYER_COMMAND_VOLUME_UP) || (*cmd == media_player::MEDIA_PLAYER_COMMAND_VOLUME_DOWN)) { ticks_to_wait = 0; // Wait 0 ticks for queue to be free, volume sets aren't that important! } xQueueSend(this->media_control_command_queue_, &media_command, ticks_to_wait); return; } } media_player::MediaPlayerTraits SpeakerMediaPlayer::get_traits() { auto traits = media_player::MediaPlayerTraits(); if (!this->single_pipeline_()) { traits.set_supports_pause(true); } #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF traits.set_supports_turn_off_on(true); #endif if (this->announcement_format_.has_value()) { traits.get_supported_formats().push_back(this->announcement_format_.value()); } if (this->media_format_.has_value()) { traits.get_supported_formats().push_back(this->media_format_.value()); } else if (this->single_pipeline_() && this->announcement_format_.has_value()) { // Only one pipeline is defined, so use the announcement format (if configured) for the default purpose media_player::MediaPlayerSupportedFormat media_format = this->announcement_format_.value(); media_format.purpose = media_player::MediaPlayerFormatPurpose::PURPOSE_DEFAULT; traits.get_supported_formats().push_back(media_format); } return traits; }; void SpeakerMediaPlayer::save_volume_restore_state_() { VolumeRestoreState volume_restore_state; volume_restore_state.volume = this->volume; volume_restore_state.is_muted = this->is_muted_; this->pref_.save(&volume_restore_state); } void SpeakerMediaPlayer::set_mute_state_(bool mute_state) { if (this->media_speaker_ != nullptr) { this->media_speaker_->set_mute_state(mute_state); } if (this->announcement_speaker_ != nullptr) { this->announcement_speaker_->set_mute_state(mute_state); } bool old_mute_state = this->is_muted_; this->is_muted_ = mute_state; this->save_volume_restore_state_(); if (old_mute_state != mute_state) { if (mute_state) { this->defer([this]() { this->mute_trigger_.trigger(); }); } else { this->defer([this]() { this->unmute_trigger_.trigger(); }); } } } void SpeakerMediaPlayer::set_volume_(float volume, bool publish) { // Remap the volume to fit with in the configured limits float bounded_volume = remap(volume, 0.0f, 1.0f, this->volume_min_, this->volume_max_); if (this->media_speaker_ != nullptr) { this->media_speaker_->set_volume(bounded_volume); } if (this->announcement_speaker_ != nullptr) { this->announcement_speaker_->set_volume(bounded_volume); } if (publish) { this->volume = volume; this->save_volume_restore_state_(); } // Turn on the mute state if the volume is effectively zero, off otherwise if (volume < 0.001) { this->set_mute_state_(true); } else { this->set_mute_state_(false); } this->defer([this, volume]() { this->volume_trigger_.trigger(volume); }); } } // namespace speaker } // namespace esphome #endif