"""Voice satellite protocol.""" import asyncio import hashlib import logging import posixpath import shutil import threading import time from collections.abc import Iterable from functools import partial from typing import Any, Dict, List, Optional, Set, Union from urllib.parse import urlparse, urlunparse from urllib.request import urlopen # pylint: disable=no-name-in-module from aioesphomeapi.api_pb2 import ( # type: ignore[attr-defined] AuthenticationRequest, DeviceInfoRequest, DeviceInfoResponse, LightCommandRequest, ListEntitiesDoneResponse, ListEntitiesRequest, MediaPlayerCommandRequest, NumberCommandRequest, SelectCommandRequest, SubscribeHomeAssistantStatesRequest, SubscribeStatesRequest, SwitchCommandRequest, VoiceAssistantAnnounceFinished, VoiceAssistantAnnounceRequest, VoiceAssistantAudio, VoiceAssistantConfigurationRequest, VoiceAssistantConfigurationResponse, VoiceAssistantEventResponse, VoiceAssistantExternalWakeWord, VoiceAssistantRequest, VoiceAssistantSetConfiguration, VoiceAssistantTimerEventResponse, VoiceAssistantWakeWord, ) from aioesphomeapi.core import MESSAGE_TYPE_TO_PROTO from aioesphomeapi.model import ( VoiceAssistantEventType, VoiceAssistantFeature, VoiceAssistantTimerEventType, ) from google.protobuf import message from pymicro_wakeword import MicroWakeWord from pyopen_wakeword import OpenWakeWord from .api_server import APIServer from .entity import ( ButtonEventSensorEntity, LEDLightEntity, MediaPlayerEntity, MicSettingEntity, MuteSwitchEntity, StopWordSensitivityNumberEntity, ThinkingSoundEntity, WakeWord1SensitivityNumberEntity, WakeWord2SensitivityNumberEntity, ) from .models import AvailableWakeWord, ServerState, WakeWordType from .peripheral_api import LVAEvent from .util import call_all _LOGGER = logging.getLogger(__name__) PROTO_TO_MESSAGE_TYPE = {v: k for k, v in MESSAGE_TYPE_TO_PROTO.items()} _HAS_AUDIO_DATA2 = "data2" in {f.name for f in VoiceAssistantAudio.DESCRIPTOR.fields} class VoiceSatelliteProtocol(APIServer): def __init__(self, state: ServerState) -> None: super().__init__(state.name) self.state = state self.state.satellite = self self.state.connected = False # Report capabilities appropriately if state.output_only: _LOGGER.debug("Output only features") self.supported_features = VoiceAssistantFeature.API_AUDIO | VoiceAssistantFeature.ANNOUNCE else: _LOGGER.debug("Voice assistant features") self.supported_features = ( VoiceAssistantFeature.VOICE_ASSISTANT | VoiceAssistantFeature.API_AUDIO | VoiceAssistantFeature.ANNOUNCE | VoiceAssistantFeature.START_CONVERSATION | VoiceAssistantFeature.TIMERS ) # Channel 1 carries echo-reference audio; advertise SPEAKER so HA knows to use it for server-side AEC. if state.audio_input_channels >= 2: self.supported_features |= VoiceAssistantFeature.MULTI_CHANNEL_AUDIO # pylint: disable=no-member existing_mute_switches = [entity for entity in self.state.entities if isinstance(entity, MuteSwitchEntity)] existing_media_players = [entity for entity in self.state.entities if isinstance(entity, MediaPlayerEntity)] if existing_media_players: # Keep the first instance and remove any extras. self.state.media_player_entity = existing_media_players[0] for extra_player in existing_media_players[1:]: self.state.entities.remove(extra_player) if existing_mute_switches: self.state.mute_switch_entity = existing_mute_switches[0] for extra_mute in existing_mute_switches[1:]: self.state.entities.remove(extra_mute) if self.state.media_player_entity is None: self.state.media_player_entity = MediaPlayerEntity( server=self, key=len(state.entities), name="Media Player", object_id="linux_voice_assistant_media_player", music_player=state.music_player, announce_player=state.tts_player, initial_volume=state.volume, ) self.state.entities.append(self.state.media_player_entity) elif self.state.media_player_entity not in self.state.entities: self.state.entities.append(self.state.media_player_entity) self.state.media_player_entity.server = self self.state.media_player_entity.volume = state.volume self.state.media_player_entity.previous_volume = state.volume # Add/update mute switch entity (like ESPHome Voice PE) mute_switch = self.state.mute_switch_entity if mute_switch is None: mute_switch = MuteSwitchEntity( server=self, key=len(state.entities), name="Mute", object_id="mute", get_muted=lambda: self.state.muted, set_muted=self._set_muted, ) self.state.entities.append(mute_switch) self.state.mute_switch_entity = mute_switch elif mute_switch not in self.state.entities: self.state.entities.append(mute_switch) mute_switch.server = self mute_switch.update_get_muted(lambda: self.state.muted) mute_switch.update_set_muted(self._set_muted) mute_switch.sync_with_state() existing_thinking_sound_switches = [entity for entity in self.state.entities if isinstance(entity, ThinkingSoundEntity)] if existing_thinking_sound_switches: self.state.thinking_sound_entity = existing_thinking_sound_switches[0] for extra_thinking in existing_thinking_sound_switches[1:]: self.state.entities.remove(extra_thinking) # Add/update thinking sound entity thinking_sound_switch = self.state.thinking_sound_entity if thinking_sound_switch is None: thinking_sound_switch = ThinkingSoundEntity( server=self, key=len(state.entities), name="Thinking Sound", object_id="thinking_sound", get_thinking_sound_enabled=lambda: self.state.thinking_sound_enabled, set_thinking_sound_enabled=self._set_thinking_sound_enabled, ) self.state.entities.append(thinking_sound_switch) self.state.thinking_sound_entity = thinking_sound_switch elif thinking_sound_switch not in self.state.entities: self.state.entities.append(thinking_sound_switch) # Load thinking sound enabled state from preferences if hasattr(self.state.preferences, "thinking_sound") and self.state.preferences.thinking_sound in (0, 1): self.state.thinking_sound_enabled = bool(self.state.preferences.thinking_sound) else: self.state.thinking_sound_enabled = False thinking_sound_switch.server = self thinking_sound_switch.update_get_thinking_sound_enabled(lambda: self.state.thinking_sound_enabled) thinking_sound_switch.update_set_thinking_sound_enabled(self._set_thinking_sound_enabled) thinking_sound_switch.sync_with_state() # Add/update Wake Word 1 sensitivity number entity sensitivity_1_entity = self.state.sensitivity_1_number_entity if sensitivity_1_entity is None: sensitivity_1_entity = WakeWord1SensitivityNumberEntity( server=self, key=len(state.entities), name="Wake Word 1 Sensitivity", object_id="wake_word_1_sensitivity", get_sensitivity=lambda: self.state.wake_word_1_threshold, set_sensitivity=self._set_sensitivity_1, initial_value=self.state.wake_word_1_threshold, ) self.state.entities.append(sensitivity_1_entity) self.state.sensitivity_1_number_entity = sensitivity_1_entity elif sensitivity_1_entity not in self.state.entities: self.state.entities.append(sensitivity_1_entity) sensitivity_1_entity.server = self sensitivity_1_entity.update_get_sensitivity(lambda: self.state.wake_word_1_threshold) sensitivity_1_entity.update_set_sensitivity(self._set_sensitivity_1) sensitivity_1_entity.sync_with_state() _LOGGER.debug("INIT: Wake Word 1 entity initialized with value %.3f", sensitivity_1_entity.value) # Add/update Wake Word 2 sensitivity number entity sensitivity_2_entity = self.state.sensitivity_2_number_entity if sensitivity_2_entity is None: sensitivity_2_entity = WakeWord2SensitivityNumberEntity( server=self, key=len(state.entities), name="Wake Word 2 Sensitivity", object_id="wake_word_2_sensitivity", get_sensitivity=lambda: self.state.wake_word_2_threshold, set_sensitivity=self._set_sensitivity_2, initial_value=self.state.wake_word_2_threshold, ) self.state.entities.append(sensitivity_2_entity) self.state.sensitivity_2_number_entity = sensitivity_2_entity elif sensitivity_2_entity not in self.state.entities: self.state.entities.append(sensitivity_2_entity) sensitivity_2_entity.server = self sensitivity_2_entity.update_get_sensitivity(lambda: self.state.wake_word_2_threshold) sensitivity_2_entity.update_set_sensitivity(self._set_sensitivity_2) sensitivity_2_entity.sync_with_state() # Add/update Stop Word sensitivity number entity stop_sensitivity_entity = self.state.stop_sensitivity_number_entity if stop_sensitivity_entity is None: stop_sensitivity_entity = StopWordSensitivityNumberEntity( server=self, key=len(state.entities), name="Stop Word Sensitivity", object_id="stop_word_sensitivity", get_sensitivity=lambda: self.state.stop_word_threshold, set_sensitivity=self._set_stop_sensitivity, initial_value=self.state.stop_word_threshold, ) self.state.entities.append(stop_sensitivity_entity) self.state.stop_sensitivity_number_entity = stop_sensitivity_entity elif stop_sensitivity_entity not in self.state.entities: self.state.entities.append(stop_sensitivity_entity) stop_sensitivity_entity.server = self stop_sensitivity_entity.update_get_sensitivity(lambda: self.state.stop_word_threshold) stop_sensitivity_entity.update_set_sensitivity(self._set_stop_sensitivity) stop_sensitivity_entity.sync_with_state() # Mic Gain if self.state.mic_gain_entity is None: self.state.mic_gain_entity = MicSettingEntity( server=self, key=len(self.state.entities), name="Mic Auto Gain", object_id="mic_gain", min_value=0.0, max_value=31.0, get_value=lambda: float(self.state.mic_auto_gain), set_value=lambda val: self.state.persist_mic_gain(float(val)), icon="mdi:microphone-plus", ) self.state.entities.append(self.state.mic_gain_entity) elif self.state.mic_gain_entity not in self.state.entities: self.state.entities.append(self.state.mic_gain_entity) self.state.mic_gain_entity.server = self self.state.mic_gain_entity.update_get_value(lambda: float(self.state.mic_auto_gain)) self.state.mic_gain_entity.update_set_value(lambda val: self.state.persist_mic_gain(float(val))) # type: ignore[arg-type] self.state.mic_gain_entity.sync_with_state() # Mic Noise Suppression _NOISE_OPTIONS = ["Off", "Low", "Medium", "High", "Max"] _NOISE_TO_INT = {label: i for i, label in enumerate(_NOISE_OPTIONS)} def _get_noise_label() -> str: return _NOISE_OPTIONS[max(0, min(4, self.state.mic_noise_suppression))] def _set_noise_label(label: Union[float, str]) -> None: self.state.persist_mic_noise(float(_NOISE_TO_INT.get(str(label), 0))) if self.state.mic_noise_suppression_entity is None: self.state.mic_noise_suppression_entity = MicSettingEntity( server=self, key=len(self.state.entities), name="Mic Noise Suppression", object_id="mic_noise", options=_NOISE_OPTIONS, get_value=_get_noise_label, set_value=_set_noise_label, icon="mdi:waveform", ) self.state.entities.append(self.state.mic_noise_suppression_entity) elif self.state.mic_noise_suppression_entity not in self.state.entities: self.state.entities.append(self.state.mic_noise_suppression_entity) self.state.mic_noise_suppression_entity.server = self self.state.mic_noise_suppression_entity.update_get_value(_get_noise_label) self.state.mic_noise_suppression_entity.update_set_value(_set_noise_label) self.state.mic_noise_suppression_entity.sync_with_state() # Mic Volume if self.state.mic_volume_entity is None: self.state.mic_volume_entity = MicSettingEntity( server=self, key=len(self.state.entities), name="Mic Volume", object_id="mic_volume", min_value=1.0, max_value=100.0, get_value=lambda: float(self.state.mic_volume), set_value=lambda val: self.state.persist_mic_volume(float(val)), icon="mdi:microphone-settings", ) self.state.entities.append(self.state.mic_volume_entity) elif self.state.mic_volume_entity not in self.state.entities: self.state.entities.append(self.state.mic_volume_entity) self.state.mic_volume_entity.server = self self.state.mic_volume_entity.update_get_value(lambda: float(self.state.mic_volume)) self.state.mic_volume_entity.update_set_value(lambda val: self.state.persist_mic_volume(float(val))) # NOTE: ButtonEventSensorEntity is NOT created here unconditionally. # It is only materialised when a peripheral sends the register_button # command (see register_pending_button below), mirroring the same # opt-in pattern used by register_light for LEDLightEntity. # Materialise the Light entities peripherals registered before # this satellite was constructed (or reattach existing ones). self.register_pending_lights() # Materialise ButtonEventSensorEntity if a peripheral already registered # button support before this satellite was constructed (e.g. on an HA # reconnect while the peripheral container stayed connected to LVA). self.register_pending_button() # ---- Instance variables ---- self._is_streaming_audio = False self._tts_url: Optional[str] = None self._tts_played = False self._continue_conversation = False self._timer_finished = False self._timer_ring_start: Optional[float] = None self._processing = False self._pipeline_active = False self._external_wake_words: Dict[str, VoiceAssistantExternalWakeWord] = {} self._disconnect_event = asyncio.Event() # ------------------------------------------------------------------ # Peripheral API helper # ------------------------------------------------------------------ def _emit( self, event: LVAEvent, data: Optional[Dict[str, Any]] = None, ) -> None: """ Emit a peripheral LED/button event. Thread-safe: delegates to ``emit_event_sync`` which uses ``run_coroutine_threadsafe`` when called from outside the event loop. """ api = self.state.peripheral_api if api is not None: api.emit_event_sync(event, data) def register_pending_lights(self) -> None: """Materialise LightEntities for peripheral registered lights. Called from __init__ so entities exist by the time HA enumerates, and again from the peripheral_api dispatcher when a light arrives after the satellite is already running. HA only sees a late registration after its next reconnect, but LVA stays consistent. """ for spec in self.state.pending_lights: if spec.object_id in self.state.led_light_entities: # Already materialised. Reattach the server in case the # satellite has been reconstructed (HA reconnect). self.state.led_light_entities[spec.object_id].server = self if self.state.led_light_entities[spec.object_id] not in self.state.entities: self.state.entities.append(self.state.led_light_entities[spec.object_id]) continue object_id = spec.object_id entity = LEDLightEntity( server=self, key=len(self.state.entities), name=spec.name, object_id=object_id, icon=spec.icon, effects=spec.effects, supports_rgb=spec.supports_rgb, supports_brightness=spec.supports_brightness, on_changed=partial(self._on_led_light_changed, object_id), ) self.state.entities.append(entity) self.state.led_light_entities[object_id] = entity def register_pending_button(self) -> None: """Materialise ButtonEventSensorEntity once a peripheral has registered button support. Called from __init__ (handles HA reconnects where the peripheral container stayed connected to LVA and pending_button is already True) and from PeripheralAPIServer._register_button() when the command arrives at runtime. Safe to call multiple times: idempotent — if the entity already exists it is only reattached to the current satellite server instance. """ if not self.state.pending_button: return if self.state.button_event_sensor_entity is not None: # Already materialised — reattach the server in case the satellite # has been reconstructed for an HA reconnect. self.state.button_event_sensor_entity.server = self if self.state.button_event_sensor_entity not in self.state.entities: self.state.entities.append(self.state.button_event_sensor_entity) return entity = ButtonEventSensorEntity( server=self, key=len(self.state.entities), name="Button Press", object_id="button_press_event", ) self.state.entities.append(entity) self.state.button_event_sensor_entity = entity _LOGGER.info("Button event sensor entity materialised") def _on_led_light_changed(self, object_id: str) -> None: """Forward an HA Light entity change to peripherals as light_command. The event carries object_id so a peripheral that registered more than one light can route it to the correct hardware. """ entity = self.state.led_light_entities.get(object_id) if entity is None: return self._emit(LVAEvent.LIGHT_COMMAND, entity.state_dict()) # ------------------------------------------------------------------ # Mute / thinking sound # ------------------------------------------------------------------ def _set_thinking_sound_enabled(self, new_state: bool) -> None: self.state.thinking_sound_enabled = bool(new_state) self.state.preferences.thinking_sound = 1 if self.state.thinking_sound_enabled else 0 if self.state.thinking_sound_enabled: _LOGGER.debug("Thinking sound enabled") else: _LOGGER.debug("Thinking sound disabled") pass self.state.save_preferences() def _set_sensitivity_1(self, new_value: float) -> None: self.state.wake_word_1_threshold = float(new_value) self.state.preferences.wake_word_1_sensitivity = float(new_value) self.state.save_preferences() _LOGGER.debug("Wake Word 1 Sensitivity value set to: %s", new_value) # Sync entity state if self.state.sensitivity_1_number_entity is not None: self.state.sensitivity_1_number_entity.sync_with_state() def _set_sensitivity_2(self, new_value: float) -> None: self.state.wake_word_2_threshold = float(new_value) self.state.preferences.wake_word_2_sensitivity = float(new_value) self.state.save_preferences() _LOGGER.debug("Wake Word 2 Sensitivity value set to: %s", new_value) # Sync entity state if self.state.sensitivity_2_number_entity is not None: self.state.sensitivity_2_number_entity.sync_with_state() def _set_stop_sensitivity(self, new_value: float) -> None: self.state.stop_word_threshold = float(new_value) self.state.preferences.stop_word_sensitivity = float(new_value) self.state.save_preferences() _LOGGER.debug("Stop Word Sensitivity value set to: %s", new_value) # Sync entity state if self.state.stop_sensitivity_number_entity is not None: self.state.stop_sensitivity_number_entity.sync_with_state() def _set_muted(self, new_state: bool) -> None: self.state.muted = bool(new_state) self._emit(LVAEvent.MUTED, {"muted": self.state.muted}) if self.state.muted: # voice_assistant.stop behavior _LOGGER.debug("Muting voice assistant (voice_assistant.stop)") self._is_streaming_audio = False self.state.tts_player.stop() # Stop any ongoing voice processing self.state.stop_word.is_active = False # type: ignore[attr-defined] self.state.tts_player.play(self.state.mute_sound) else: # voice_assistant.start_continuous behavior _LOGGER.debug("Unmuting voice assistant (voice_assistant.start_continuous)") self.state.tts_player.play(self.state.unmute_sound) self._emit(LVAEvent.IDLE) # ------------------------------------------------------------------ # Voice pipeline event handler # ------------------------------------------------------------------ def handle_voice_event(self, event_type: VoiceAssistantEventType, data: Dict[str, str]) -> None: _LOGGER.debug("Voice event: type=%s, data=%s", event_type.name, data) if event_type == VoiceAssistantEventType.VOICE_ASSISTANT_RUN_START: self._tts_url = data.get("url") self._tts_played = False self._continue_conversation = False self._pipeline_active = True elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_START: self._emit(LVAEvent.THINKING) # Play optional audible thinking sound if self.state.thinking_sound_enabled: processing = getattr(self.state, "processing_sound", None) if processing: _LOGGER.debug("Playing processing sound: %s", processing) self.state.stop_word.is_active = True # type: ignore[attr-defined] self._processing = True self.duck() self.state.tts_player.play(self.state.processing_sound) elif event_type in ( VoiceAssistantEventType.VOICE_ASSISTANT_STT_VAD_END, VoiceAssistantEventType.VOICE_ASSISTANT_STT_END, ): self._is_streaming_audio = False if event_type == VoiceAssistantEventType.VOICE_ASSISTANT_STT_END: stt_text = data.get("text", "").strip() if stt_text: self._emit(LVAEvent.STT_TEXT, {"text": stt_text}) _LOGGER.debug("STT transcript: %s", stt_text) elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_PROGRESS: if data.get("tts_start_streaming") == "1": # Start streaming early self.play_tts() elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_INTENT_END: if data.get("continue_conversation") == "1": self._continue_conversation = True elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_TTS_START: tts_text = data.get("text", "").strip() if tts_text: self._emit(LVAEvent.TTS_TEXT, {"text": tts_text}) _LOGGER.debug("TTS response text: %s", tts_text) elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_TTS_END: self._tts_url = data.get("url") self.play_tts() elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_RUN_END: self._is_streaming_audio = False if not self._tts_played: self._pipeline_active = False self._tts_finished() # When TTS is playing, keep _pipeline_active = True to block # false wake word detections from speaker audio feedback. # _tts_finished() callback will clear it when playback ends. self._tts_played = False elif event_type == VoiceAssistantEventType.VOICE_ASSISTANT_ERROR: self._emit(LVAEvent.PIPELINE_ERROR) # ------------------------------------------------------------------ # Timer event handler # ------------------------------------------------------------------ def handle_timer_event( self, event_type: VoiceAssistantTimerEventType, msg: VoiceAssistantTimerEventResponse, ) -> None: _LOGGER.debug("Timer event: type=%s", event_type.name) # Build countdown data from the protobuf message fields. # total_seconds: the original timer duration. # seconds_left: remaining seconds at the time of this event. timer_data = { "id": msg.timer_id, "name": msg.name, "total_seconds": msg.total_seconds, "seconds_left": msg.seconds_left, } if event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_STARTED: self._emit(LVAEvent.TIMER_TICKING, timer_data) elif event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_UPDATED: self._emit(LVAEvent.TIMER_UPDATED, timer_data) elif event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_CANCELLED: self._emit(LVAEvent.IDLE) elif event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_FINISHED: if not self._timer_finished: self.state.active_wake_words.add(self.state.stop_word.id) self._timer_finished = True self._timer_ring_start = time.monotonic() self.duck() self._emit(LVAEvent.TIMER_RINGING, timer_data) self._play_timer_finished() # ------------------------------------------------------------------ # Message routing # ------------------------------------------------------------------ def handle_message(self, msg: message.Message) -> Iterable[message.Message]: # noqa: C901 (acceptable complexity for a message router) if isinstance(msg, VoiceAssistantEventResponse): # Pipeline event data: Dict[str, str] = {} for arg in msg.data: data[arg.name] = arg.value self.handle_voice_event(VoiceAssistantEventType(msg.event_type), data) elif isinstance(msg, VoiceAssistantAnnounceRequest): _LOGGER.debug("Announcing: %s", msg.text) assert self.state.media_player_entity is not None urls = [] if msg.preannounce_media_id: urls.append(msg.preannounce_media_id) urls.append(msg.media_id) self.state.active_wake_words.add(self.state.stop_word.id) self._continue_conversation = msg.start_conversation self.duck() self._emit(LVAEvent.TTS_SPEAKING) self.state.tts_player.play(urls, done_callback=self._tts_finished) elif isinstance(msg, VoiceAssistantTimerEventResponse): self.handle_timer_event(VoiceAssistantTimerEventType(msg.event_type), msg) elif isinstance(msg, DeviceInfoRequest): _LOGGER.debug("Device info request") yield DeviceInfoResponse( uses_password=False, name=self.state.name, friendly_name=self.state.friendly_name, project_name="Open Home Foundation.Linux Voice Assistant", project_version=self.state.version, esphome_version=self.state.esphome_version, mac_address=self.state.mac_address, manufacturer="Open Home Foundation", model="Linux Voice Assistant", voice_assistant_feature_flags=self.supported_features, ) elif isinstance(msg, SubscribeStatesRequest): # Standard ESPHome state subscription. Replay current entity state to # the subscribing client. (Entities answer SubscribeHomeAssistantStatesRequest; # initial state was previously only sent as a side effect of auth.) for entity in self.state.entities: yield from entity.handle_message(SubscribeHomeAssistantStatesRequest()) elif isinstance( msg, (ListEntitiesRequest, SubscribeHomeAssistantStatesRequest, MediaPlayerCommandRequest, SwitchCommandRequest, NumberCommandRequest, SelectCommandRequest, LightCommandRequest), ): for entity in self.state.entities: yield from entity.handle_message(msg) # Emit peripheral event when background music starts playing. # Announcements (TTS) are explicitly excluded — those are covered # by TTS_SPEAKING. if isinstance(msg, MediaPlayerCommandRequest) and msg.has_media_url: is_announcement = msg.has_announcement and msg.announcement if not is_announcement: self._emit(LVAEvent.MEDIA_PLAYER_PLAYING) if isinstance(msg, ListEntitiesRequest): yield ListEntitiesDoneResponse() elif isinstance(msg, VoiceAssistantConfigurationRequest): _LOGGER.debug("✅ Received VoiceAssistantConfigurationRequest from Home Assistant") _LOGGER.debug(" -> Request contains %d external wake words", len(msg.external_wake_words)) available_wake_words = [ VoiceAssistantWakeWord( id=ww.id, wake_word=ww.wake_word, trained_languages=ww.trained_languages, ) for ww in self.state.available_wake_words.values() ] # Log available internal wake words first internal_ww_count = len(self.state.available_wake_words) _LOGGER.debug(" -> Found %d internal available wake words", internal_ww_count) for ww in available_wake_words: _LOGGER.debug(" - %s: '%s' (langs: %s)", ww.id, ww.wake_word, ww.trained_languages) for eww in msg.external_wake_words: _LOGGER.debug(" -> Processing external wake word: id=%s, word='%s', type=%s", eww.id, eww.wake_word, eww.model_type) if eww.model_type != "micro": _LOGGER.debug(" → Skipping: not micro model type") continue _LOGGER.debug(" → Adding to available wake words") available_wake_words.append( VoiceAssistantWakeWord( id=eww.id, wake_word=eww.wake_word, trained_languages=eww.trained_languages, ) ) self._external_wake_words[eww.id] = eww _LOGGER.debug(" → Stored in external wake words cache") active_ww_ids = [ww.id for ww in self.state.wake_words.values() if ww.id in self.state.active_wake_words] _LOGGER.debug(" -> Active wake word IDs: %s", active_ww_ids) yield VoiceAssistantConfigurationResponse( available_wake_words=available_wake_words, active_wake_words=active_ww_ids, max_active_wake_words=2, ) _LOGGER.info("✅ Connected to Home Assistant - Configuration handshake completed") _LOGGER.debug("✅ VoiceAssistantConfigurationResponse sent successfully") elif isinstance(msg, VoiceAssistantSetConfiguration): # Change active wake words active_wake_words: Set[str] = set() new_wake_words: List[Optional[str]] = [None, None] # Get old positions before modification old_positions: Dict[str, int] = {} for idx, ww_id in enumerate(self.state.preferences.active_wake_words): if ww_id is not None and idx < 2: old_positions[ww_id] = idx # Process new active wake words for wake_word_id in msg.active_wake_words: if wake_word_id in self.state.wake_words: # Already active active_wake_words.add(wake_word_id) else: model_info = self.state.available_wake_words.get(wake_word_id) if not model_info: # Check external wake words (may require download) external_wake_word = self._external_wake_words.get(wake_word_id) if not external_wake_word: continue model_info = self._download_external_wake_word(external_wake_word) if not model_info: continue self.state.available_wake_words[wake_word_id] = model_info _LOGGER.debug("Loading wake word: %s", model_info.wake_word_path) self.state.wake_words[wake_word_id] = model_info.load() _LOGGER.info("Wake word set: %s", wake_word_id) active_wake_words.add(wake_word_id) # Keep old positions remaining_ww = list(active_wake_words) placed = set() # First, place Wake Words in their old positions. for ww_id in remaining_ww: if ww_id in old_positions: pos = old_positions[ww_id] if pos < 2: new_wake_words[pos] = ww_id placed.add(ww_id) # Add remaining wake words to free slots free_slots = [i for i in range(2) if new_wake_words[i] is None] for ww_id in remaining_ww: if ww_id not in placed and free_slots: pos = free_slots.pop(0) new_wake_words[pos] = ww_id placed.add(ww_id) # If only one wake word is left and it was at position 1, position 0 remains None # Position 2 automatically stays None if not occupied self.state.active_wake_words = active_wake_words _LOGGER.debug("Active wake words: %s", active_wake_words) _LOGGER.debug("Wake word positions: [0]=%s, [1]=%s", new_wake_words[0], new_wake_words[1]) self.state.preferences.active_wake_words = new_wake_words self.state.save_preferences() self.state.wake_words_changed = True # ------------------------------------------------------------------ # Audio streaming # ------------------------------------------------------------------ # handle_audio — both channels in ONE message def handle_audio(self, audio_chunk: bytes, audio_chunk_2: Optional[bytes] = None) -> None: if not self._is_streaming_audio or self.state.muted: return if _HAS_AUDIO_DATA2 and audio_chunk_2 is not None: self.send_messages([VoiceAssistantAudio(data=audio_chunk, data2=audio_chunk_2)]) else: self.send_messages([VoiceAssistantAudio(data=audio_chunk)]) # ------------------------------------------------------------------ # Wake word / stop # ------------------------------------------------------------------ def wakeup(self, wake_word: Union[MicroWakeWord, OpenWakeWord]) -> None: if self.state.muted: # Don't respond to wake words when muted (voice_assistant.stop behavior) return if self._pipeline_active: _LOGGER.debug("Ignoring wake word - pipeline already active") return wake_word_phrase = wake_word.wake_word # type: ignore[union-attr] _LOGGER.debug("Detected wake word: %s", wake_word_phrase) self._timer_finished = False self._timer_ring_start = None _LOGGER.debug("Stopping timer finished sound") self._pipeline_active = True self._emit(LVAEvent.WAKE_WORD_DETECTED) self.duck() self.state.tts_player.play( self.state.wakeup_sound, done_callback=lambda: self._on_wakeup_sound_finished(wake_word_phrase), ) def _on_wakeup_sound_finished(self, wake_word_phrase: str) -> None: """Callback invoked when the wakeup chime finishes; begin STT streaming.""" _LOGGER.debug( "Wakeup sound finished, starting audio streaming for: %s", wake_word_phrase, ) self.send_messages([VoiceAssistantRequest(start=True, wake_word_phrase=wake_word_phrase)]) self._is_streaming_audio = True self._emit(LVAEvent.LISTENING) def start_listening(self) -> None: """ Manually start the voice pipeline from a button press. Plays ``start_listening_sound`` first, then sends ``VoiceAssistantRequest`` and begins streaming audio — identical flow to ``wakeup()`` but without a wake-word phrase and using the dedicated button-press sound instead of the wake-word chime. Also stops ringing timer. """ if self.state.muted: return if self._pipeline_active: _LOGGER.debug("Ignoring start_listening - pipeline already active") return _LOGGER.debug("Button start_listening triggered") self._timer_finished = False self._timer_ring_start = None _LOGGER.debug("Stopping timer finished sound") self._pipeline_active = True self.duck() self.state.tts_player.play( self.state.start_listening_sound, done_callback=self._on_start_listening_sound_finished, ) def _on_start_listening_sound_finished(self) -> None: """Callback invoked when the start-listening chime finishes; begin STT streaming.""" _LOGGER.debug("Start-listening sound finished, starting audio streaming") self.send_messages([VoiceAssistantRequest(start=True, wake_word_phrase="")]) self._is_streaming_audio = True self._emit(LVAEvent.LISTENING) def stop(self) -> None: self.state.active_wake_words.discard(self.state.stop_word.id) self._pipeline_active = False if self._timer_finished: self._timer_finished = False self._timer_ring_start = None self.unduck() self.state.tts_player.stop() self._emit(LVAEvent.IDLE) _LOGGER.debug("Stopping timer finished sound") else: # tts_player.stop() invokes the done_callback (_tts_finished), # so we don't call _tts_finished() again explicitly. self.state.tts_player.stop() _LOGGER.debug("TTS response stopped manually") # ------------------------------------------------------------------ # TTS # ------------------------------------------------------------------ def play_tts(self) -> None: if (not self._tts_url) or self._tts_played: return self._tts_played = True _LOGGER.debug("Playing TTS response: %s", self._tts_url) self.state.active_wake_words.add(self.state.stop_word.id) self._emit(LVAEvent.TTS_SPEAKING) self.state.tts_player.play(self._tts_url, done_callback=self._tts_finished) def _tts_finished(self) -> None: self._pipeline_active = False self.state.active_wake_words.discard(self.state.stop_word.id) self.send_messages([VoiceAssistantAnnounceFinished()]) self._emit(LVAEvent.TTS_FINISHED) if self._continue_conversation: self._continue_conversation = False # Keep pipeline active during the settle delay so the mic stays closed # and does not capture the tail end of the TTS audio from the speaker. self._pipeline_active = True self._emit(LVAEvent.LISTENING) _LOGGER.debug("Continuing conversation after %.2fs settle delay", self.state.continue_conversation_delay) def _start_continued_conversation() -> None: if self.state.muted: _LOGGER.debug("Skipping continued conversation: muted") self._pipeline_active = False self.unduck() return self.send_messages([VoiceAssistantRequest(start=True)]) self._is_streaming_audio = True _LOGGER.debug("Continued conversation started") threading.Timer(self.state.continue_conversation_delay, _start_continued_conversation).start() else: self._continue_conversation = False self.unduck() self._emit(LVAEvent.IDLE) _LOGGER.debug("TTS response finished") # ------------------------------------------------------------------ # Ducking # ------------------------------------------------------------------ def duck(self) -> None: _LOGGER.debug("Ducking music") self.state.music_player.duck() def unduck(self) -> None: _LOGGER.debug("Unducking music") self.state.music_player.unduck() # ------------------------------------------------------------------ # Timer finished loop # ------------------------------------------------------------------ def _play_timer_finished(self) -> None: if not self._timer_finished: _LOGGER.debug("Timer finished sound stopped") self.unduck() self._timer_ring_start = None return # Auto-stop after timer_max_ring_seconds if self._timer_ring_start is not None: elapsed = time.monotonic() - self._timer_ring_start if elapsed >= self.state.timer_max_ring_seconds: _LOGGER.info( "Timer auto-stopped after %.0f seconds (max=%.0f)", elapsed, self.state.timer_max_ring_seconds, ) self._timer_finished = False self._timer_ring_start = None self.state.active_wake_words.discard(self.state.stop_word.id) self.unduck() return self.state.tts_player.play( self.state.timer_finished_sound, done_callback=lambda: call_all( lambda: time.sleep(1.0), self._play_timer_finished, ), ) def connection_made(self, transport) -> None: super().connection_made(transport) # Track every live connection so asynchronous entity-state changes can be # broadcast to all of them (see ServerState.broadcast). if self not in self.state.connections: self.state.connections.append(self) # ------------------------------------------------------------------ # Connection lifecycle # ------------------------------------------------------------------ def connection_lost(self, exc: Optional[Exception]) -> None: super().connection_lost(exc) self._disconnect_event.set() self._is_streaming_audio = False self._tts_url = None self._tts_played = False self._continue_conversation = False self._timer_finished = False self._pipeline_active = False # Deregister this connection. if self in self.state.connections: self.state.connections.remove(self) # Only tear down shared playback/state when the LAST client disconnects. # Otherwise a secondary client (a diagnostic tool, a second dashboard, or # Home Assistant's own overlapping reconnect) dropping would stop audio # that belongs to a client still connected. if not self.state.connections: # Stop any ongoing audio playback and wake/stop word processing. try: self.state.music_player.stop() except Exception: # pylint: disable=broad-except _LOGGER.exception("Failed to stop music player during disconnect") try: self.state.tts_player.stop() except Exception: # pylint: disable=broad-except _LOGGER.exception("Failed to stop TTS player during disconnect") self.state.stop_word.is_active = False # type: ignore[attr-defined] self.state.connected = False if self.state.satellite is self: self.state.satellite = None if self.state.mute_switch_entity is not None: self.state.mute_switch_entity.sync_with_state() if self.state.mic_gain_entity is not None: self.state.mic_gain_entity.sync_with_state() if self.state.mic_noise_suppression_entity is not None: self.state.mic_noise_suppression_entity.sync_with_state() if self.state.mic_volume_entity is not None: self.state.mic_volume_entity.sync_with_state() # Notify peripheral container that HA is no longer reachable self._emit(LVAEvent.DISCONNECTED) _LOGGER.info("Disconnected from Home Assistant; waiting for reconnection") def process_packet(self, msg_type: int, packet_data: bytes) -> None: super().process_packet(msg_type, packet_data) if msg_type == PROTO_TO_MESSAGE_TYPE[AuthenticationRequest]: self.state.connected = True _LOGGER.debug("Authentication successful, connected to Home Assistant") # Send states after connect states: List[message.Message] = [] _LOGGER.debug("Found %d entities in state", len(self.state.entities)) for i, entity in enumerate(self.state.entities): entity_states = list(entity.handle_message(SubscribeHomeAssistantStatesRequest())) states.extend(entity_states) _LOGGER.debug("Entity %d (%s) returned %d state messages", i, type(entity).__name__, len(entity_states)) _LOGGER.debug("Total state messages to send: %d", len(states)) self.send_messages(states) for i, msg in enumerate(states): _LOGGER.debug("Sent state message %d: %s", i, type(msg).__name__) _LOGGER.debug("All entity states sent after connect") # Notify peripherals that Home Assistant is now connected self._emit(LVAEvent.ZEROCONF, {"status": "connected"}) # ------------------------------------------------------------------ # External wake word download # ------------------------------------------------------------------ def _download_external_wake_word(self, external_wake_word: VoiceAssistantExternalWakeWord) -> Optional[AvailableWakeWord]: eww_dir = self.state.download_dir / "external_wake_words" eww_dir.mkdir(parents=True, exist_ok=True) config_path = eww_dir / f"{external_wake_word.id}.json" should_download_config = not config_path.exists() # Check if we need to download the model file model_path = eww_dir / f"{external_wake_word.id}.tflite" should_download_model = True if model_path.exists(): model_size = model_path.stat().st_size if model_size == external_wake_word.model_size: with open(model_path, "rb") as model_file: model_hash = hashlib.sha256(model_file.read()).hexdigest() if model_hash == external_wake_word.model_hash: should_download_model = False _LOGGER.debug( "Model size and hash match for %s. Skipping download.", external_wake_word.id, ) if should_download_config or should_download_model: # Download config _LOGGER.debug("Downloading %s to %s", external_wake_word.url, config_path) with urlopen(external_wake_word.url) as request: if request.status != 200: _LOGGER.warning( "Failed to download: %s, status=%s", external_wake_word.url, request.status, ) return None with open(config_path, "wb") as model_file: shutil.copyfileobj(request, model_file) if should_download_model: # Download model file parsed_url = urlparse(external_wake_word.url) parsed_url = parsed_url._replace( path=posixpath.join(posixpath.dirname(parsed_url.path), model_path.name), ) model_url = urlunparse(parsed_url) _LOGGER.debug("Downloading %s to %s", model_url, model_path) with urlopen(model_url) as request: if request.status != 200: _LOGGER.warning("Failed to download: %s, status=%s", model_url, request.status) return None with open(model_path, "wb") as model_file: shutil.copyfileobj(request, model_file) return AvailableWakeWord( id=external_wake_word.id, type=WakeWordType.MICRO_WAKE_WORD, wake_word=external_wake_word.wake_word, trained_languages=external_wake_word.trained_languages, wake_word_path=config_path, )