From 3ea32895ee256712ddf889c5e13db743afd2a6a3 Mon Sep 17 00:00:00 2001 From: Omar Amin <65174210+omaramin-2000@users.noreply.github.com> Date: Tue, 21 Apr 2026 02:04:11 +0200 Subject: [PATCH] Wake word sensitivity (#207) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Introduces configurable wake word and stop word sensitivity - Adds UI-controllable thresholds for wake word detection - Refactors wake word loading and handling logic - Improves audio preprocessing and runtime behavior ## Key Features - Per-wake-word sensitivity configuration (Wake Word 1 & 2) - Configurable stop word sensitivity - Fine-grained numeric control (0.0 – 1.0, step 0.001) - Sensitivity values exposed as Number Entities for Home Assistant - Persistent storage of user-defined sensitivity values ## Wake Word Handling - Support for model-specific default thresholds (`probability_cutoff`) - Automatic fallback to defaults if no user value is set - Separate thresholds for: - Wake Word 1 - Wake Word 2 - Stop Word - Unified handling for MicroWakeWord and OpenWakeWord models ## UI / API Integration - Adds new Home Assistant entities: - `WakeWord1SensitivityNumberEntity` - `WakeWord2SensitivityNumberEntity` - `StopWordSensitivityNumberEntity` - Entities support: - Live updates - Persistence via preferences - Synchronization with runtime state - Pushes updated values to Home Assistant on startup ## Refactoring - Extracted wake word logic into `wake_word.py` - Model discovery - Model loading - Stop word handling - Simplified `__main__.py` by delegating wake word responsibilities - Improved separation between configuration and runtime state ## Additional Changes - Improved debug logging across initialization and runtime ## Impact - Enables fine-tuning of wake word detection accuracy - Reduces false positives and missed triggers - Improves usability via UI-based configuration - Provides better flexibility across different environments and microphones --- README.md | 3 +- docker-compose.dev.yml | 8 +- docs/audio_options.md | 26 ++- linux_voice_assistant/__main__.py | 231 ++++++++++++------- linux_voice_assistant/entity.py | 191 +++++++++++++++ linux_voice_assistant/models.py | 19 +- linux_voice_assistant/satellite.py | 196 ++++++++++++++-- linux_voice_assistant/wake_word.py | 202 ++++++++++++++++ mypy.ini | 5 + wakewords/openWakeWord/alexa_v0.1.json | 5 +- wakewords/openWakeWord/hey_jarvis_v0.1.json | 5 +- wakewords/openWakeWord/hey_mycroft_v0.1.json | 5 +- wakewords/openWakeWord/hey_rhasspy_v0.1.json | 5 +- wakewords/openWakeWord/ok_nabu_v0.1.json | 7 +- 14 files changed, 793 insertions(+), 115 deletions(-) create mode 100644 linux_voice_assistant/wake_word.py diff --git a/README.md b/README.md index a129433..0b06d22 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,8 @@ usage: __main__.py [-h] [--name NAME] [--audio-input-device AUDIO_INPUT_DEVICE] | `--debug` | Print DEBUG messages to console | False | | `--output-only` | Enable output only mode | False | -💡 **Note:** There is a detailed explanation on the gain and noise suppression flags in the [audio options](docs/audio_options.md) file. +💡 **Note:** There is a detailed explanation on the gain, noise suppression, and wake word sensitivity flags in the [audio options](docs/audio_options.md) file. + ## Build Information diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4a37a5c..aee2560 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,7 +2,7 @@ services: # Fix permissions for wakeword_data and configuration volume fix-permissions: - image: "ghcr.io/ohf-voice/linux-voice-assistant:latest" + image: "ghcr.io/ohf-voice/linux-voice-assistant-pr:207" entrypoint: [] command: "chown -R ${LVA_USER_ID:-1000}:${LVA_USER_GROUP:-1000} /app/local /app/configuration /app/wakewords/custom /app/sounds/custom" environment: @@ -24,7 +24,7 @@ services: # Linux Voice Assistant linux-voice-assistant: container_name: linux-voice-assistant - image: "ghcr.io/ohf-voice/linux-voice-assistant:latest" + image: "ghcr.io/ohf-voice/linux-voice-assistant-pr:207" restart: unless-stopped network_mode: "host" user: "${LVA_USER_ID:-1000}:${LVA_USER_GROUP:-1000}" @@ -66,7 +66,9 @@ services: timeout: 5s retries: 3 start_period: 90s - #command: + command: + - "--mic-noise-suppression" + - "1" # - --additional-parameters-here # - --but-use-env-if-possible diff --git a/docs/audio_options.md b/docs/audio_options.md index 0affe19..da5ac6a 100644 --- a/docs/audio_options.md +++ b/docs/audio_options.md @@ -1,4 +1,4 @@ -# Audio Options: Gain and Noise Suppression +# Audio Options: Gain, Noise Suppression, and Wake Word Sensitivity ## What do Gain and Noise Suppression do @@ -47,3 +47,27 @@ LVA exposes two slider entities to change these gain and noise suppression from 💡 **Note:** Setting the flag and ENV values to 0 turns them off and are not used. 💡 **Note:** Keep in mind that when the flags are set they will overwrite the previous value in the preferences file and also they will be overridden if the value is changed in Home Assistant(Also applies to the ENV file but the ENV file will always overwrite the last value on startup). + + +## Wake Word Sensitivity + +LVA exposes three numeric controls in the Home Assistant device page for fine-grained sensitivity tuning. These let you dial in the exact probability threshold that best matches your microphone quality, room acoustics, and false-activation tolerance. + +| Entity | Description | Default | +|--------|-------------|---------| +| **Wake Word 1 Sensitivity** | Probability cutoff for the primary wake word | From model manifest | +| **Wake Word 2 Sensitivity** | Probability cutoff for the secondary wake word (if active) | From model manifest | +| **Stop Word Sensitivity** | Probability cutoff for the stop word | From model manifest | + +Values range from `0.0` to `1.0`: + +- **Higher value** (e.g. `0.95`) → more selective, fewer false activations, may miss quieter or accented speech +- **Lower value** (e.g. `0.50`) → more responsive, but more likely to trigger on similar-sounding words +The defaults are read directly from each model's `.json` manifest file (the `probability_cutoff` field), so bundled models like `okay_nabu` (0.85), `hey_jarvis` (0.97), and custom downloaded models all start at their author-tested baseline. Changes made in the Home Assistant UI are persisted in `preferences.json` and survive restarts. + +**Tuning guide:** + +- If the wake word **rarely activates** (misses your voice): lower the value slightly, e.g. by `0.05` steps +- If the wake word **activates too often** (false triggers from TV, music, or similar words): raise the value slightly +- Start from the model's default and make small adjustments — a change of `0.05–0.10` is usually enough to notice a difference +- Far-field microphones with noise cancellation (e.g. ReSpeaker, Satellite1) generally work well at the default values; basic USB microphones may need a lower threshold \ No newline at end of file diff --git a/linux_voice_assistant/__main__.py b/linux_voice_assistant/__main__.py index 9860f2e..6c93e92 100644 --- a/linux_voice_assistant/__main__.py +++ b/linux_voice_assistant/__main__.py @@ -9,15 +9,16 @@ import threading import time from pathlib import Path from queue import Queue -from typing import Dict, List, Optional, Set, Union +from typing import List, Optional, Union import numpy as np import soundcard as sc +from aioesphomeapi.api_pb2 import NumberStateResponse # type: ignore # pylint: disable=no-name-in-module from getmac import get_mac_address # type: ignore from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures -from .models import AvailableWakeWord, Preferences, ServerState, WakeWordType +from .models import Preferences, ServerState from .mpv_player import MpvMediaPlayer from .satellite import VoiceSatelliteProtocol from .util import ( @@ -26,6 +27,7 @@ from .util import ( get_esphome_version, get_version, ) +from .wake_word import find_available_wake_words, load_stop_model, load_wake_models from .webrtc import WebRTCProcessor from .zeroconf import HomeAssistantZeroconf @@ -253,32 +255,7 @@ async def main() -> None: # Load available wake words wake_word_dirs = [Path(ww_dir) for ww_dir in args.wake_word_dir] wake_word_dirs.append(args.download_dir / "external_wake_words") - available_wake_words: Dict[str, AvailableWakeWord] = {} - - for wake_word_dir in wake_word_dirs: - for model_config_path in wake_word_dir.glob("*.json"): - model_id = model_config_path.stem - if model_id == args.stop_model: - # Don't show stop model as an available wake word - continue - - with open(model_config_path, "r", encoding="utf-8") as model_config_file: - model_config = json.load(model_config_file) - model_type = WakeWordType(model_config["type"]) - if model_type == WakeWordType.OPEN_WAKE_WORD: - wake_word_path = model_config_path.parent / model_config["model"] - else: - wake_word_path = model_config_path - - available_wake_words[model_id] = AvailableWakeWord( - id=model_id, - type=WakeWordType(model_type), - wake_word=model_config["wake_word"], - trained_languages=model_config.get("trained_languages", []), - wake_word_path=wake_word_path, - ) - - _LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys()))) + available_wake_words = find_available_wake_words(wake_word_dirs, args.stop_model) # Load preferences preferences_path = Path(args.preferences_file) @@ -312,40 +289,10 @@ async def main() -> None: preferences.mic_noise_suppression = args.mic_noise_suppression # Load wake/stop models - active_wake_words: Set[str] = set() - wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {} - if preferences.active_wake_words: - # Load preferred models - for wake_word_id in preferences.active_wake_words: - wake_word = available_wake_words.get(wake_word_id) - if wake_word is None: - _LOGGER.warning("Unrecognized wake word id: %s", wake_word_id) - continue - - _LOGGER.debug("Loading wake model: %s", wake_word_id) - wake_models[wake_word_id] = wake_word.load() - active_wake_words.add(wake_word_id) - - if not wake_models: - # Load default model - wake_word_id = args.wake_model - wake_word = available_wake_words[wake_word_id] - - _LOGGER.debug("Loading wake model: %s", wake_word_id) - wake_models[wake_word_id] = wake_word.load() - active_wake_words.add(wake_word_id) + wake_models, active_wake_words, fallback_used = load_wake_models(available_wake_words, [word for word in preferences.active_wake_words if word is not None], args.wake_model) # TODO: allow openWakeWord for "stop" - stop_model: Optional[MicroWakeWord] = None - for wake_word_dir in wake_word_dirs: - stop_config_path = wake_word_dir / f"{args.stop_model}.json" - if not stop_config_path.exists(): - continue - - _LOGGER.debug("Loading stop model: %s", stop_config_path) - stop_model = MicroWakeWord.from_config(stop_config_path) - break - + stop_model = load_stop_model(wake_word_dirs, args.stop_model) assert stop_model is not None state = ServerState( @@ -381,6 +328,15 @@ async def main() -> None: timer_max_ring_seconds=args.timer_max_ring_seconds, ) + if fallback_used: + # Fallback to the default model was used, save as active wake words + _LOGGER.debug("Fallback was used, save default wake words in Preferences.") + state.preferences.active_wake_words = list(active_wake_words) + state.active_wake_words = active_wake_words + state.wake_words = wake_models + state.save_preferences() + state.wake_words_changed = True + if args.enable_thinking_sound or args.mic_auto_gain or args.mic_noise_suppression: state.save_preferences() @@ -393,6 +349,22 @@ async def main() -> None: attempt = 1 server = None + # Validate VoiceSatelliteProtocol initialization BEFORE starting server + # This catches errors like missing imports or broken initialization immediately + # instead of failing silently only when first client connects + _LOGGER.debug("Validating VoiceSatelliteProtocol initialization...") + try: + # Create test instance to run complete __init__ code path + test_protocol = VoiceSatelliteProtocol(state) + # Cleanup state reference + test_protocol.state.satellite = None + del test_protocol + _LOGGER.debug("✅ VoiceSatelliteProtocol validation successful") + except Exception: + _LOGGER.critical("❌ FATAL ERROR in VoiceSatelliteProtocol initialization!", exc_info=True) + _LOGGER.critical("Program will exit immediately - fix the error above first!") + sys.exit(1) + while attempt <= max_attempts: try: server = await loop.create_server(lambda: VoiceSatelliteProtocol(state), host=host_ip_address, port=args.port) @@ -458,26 +430,6 @@ def process_audio(state: ServerState, mic, block_size: int): # little-endian 16-bit signed mic_vol_scalar = max(0.1, min(1.0, state.mic_volume / 100.0)) audio_chunk = (np.clip(audio_chunk_array * mic_vol_scalar, -1.0, 1.0) * 32767.0).astype(" 0.5: + if prob > threshold: + _LOGGER.debug("Wake word '%s' activated (probability %.3f exceeded threshold %.3f)", wake_word.wake_word, prob, threshold) # type: ignore[attr-defined] activated = True if activated and not state.muted: @@ -520,10 +578,19 @@ def process_audio(state: ServerState, mic, block_size: int): if (last_active is None) or ((now - last_active) > state.refractory_seconds): state.satellite.wakeup(wake_word) last_active = now + # Always process to keep state correct stopped = False + + # No debugging when no detection + state.stop_word.debug_probabilities = False + + # Apply stop word sensitivity threshold + state.stop_word.probability_cutoff = state.stop_word_threshold + # _LOGGER.debug("Set stop word probability cutoff to %.3f", state.stop_word_threshold) for micro_input in micro_inputs: if state.stop_word.process_streaming(micro_input): + state.stop_word.debug_probabilities = True stopped = True if stopped and (state.stop_word.id in state.active_wake_words) and not state.muted: diff --git a/linux_voice_assistant/entity.py b/linux_voice_assistant/entity.py index bda2564..ec7f1c8 100644 --- a/linux_voice_assistant/entity.py +++ b/linux_voice_assistant/entity.py @@ -25,6 +25,7 @@ from aioesphomeapi.model import ( MediaPlayerCommand, MediaPlayerEntityFeature, MediaPlayerState, + NumberMode, ) from google.protobuf import message @@ -440,3 +441,193 @@ class MicSettingEntity(ESPHomeEntity): def update_set_value(self, set_value: Callable[[Union[float, str]], None]) -> None: self._set_value = set_value + + +# ----------------------------------------------------------------------------- + + +class WakeWord1SensitivityNumberEntity(ESPHomeEntity): + def __init__( + self, + server: APIServer, + key: int, + name: str, + object_id: str, + get_sensitivity: Callable[[], float], + set_sensitivity: Callable[[float], None], + initial_value: float = 0.5, + ) -> None: + ESPHomeEntity.__init__(self, server) + + self.key = key + self.name = name + self.object_id = object_id + self._get_sensitivity = get_sensitivity + self._set_sensitivity = set_sensitivity + self.value = initial_value + self._log = logging.getLogger(f"{self.__class__.__name__}[{self.key}]") + + def update_get_sensitivity(self, get_sensitivity: Callable[[], float]) -> None: + self._get_sensitivity = get_sensitivity + + def update_set_sensitivity(self, set_sensitivity: Callable[[float], None]) -> None: + self._set_sensitivity = set_sensitivity + + def sync_with_state(self) -> None: + old_value = self.value + self.value = self._get_sensitivity() + self._log.debug("Entity synchronized: old=%.3f new=%.3f", old_value, self.value) + + def handle_message(self, msg: message.Message) -> Iterable[message.Message]: + if isinstance(msg, NumberCommandRequest) and (msg.key == self.key): + new_value = float(msg.state) + self._log.debug("Sensitivity value changed: %s => %s", self.value, new_value) + self.value = new_value + self._set_sensitivity(new_value) + yield NumberStateResponse(key=self.key, state=self.value) + elif isinstance(msg, ListEntitiesRequest): + yield ListEntitiesNumberResponse( + object_id=self.object_id, + key=self.key, + name=self.name, + entity_category=EntityCategory.CONFIG, + min_value=0.0, + max_value=1.0, + step=0.001, + mode=NumberMode.BOX, + ) + elif isinstance(msg, SubscribeHomeAssistantStatesRequest): + self.sync_with_state() + yield NumberStateResponse(key=self.key, state=self.value) + + +class WakeWord2SensitivityNumberEntity(ESPHomeEntity): + def __init__( + self, + server: APIServer, + key: int, + name: str, + object_id: str, + get_sensitivity: Callable[[], float], + set_sensitivity: Callable[[float], None], + initial_value: float = 0.5, + ) -> None: + ESPHomeEntity.__init__(self, server) + + self.key = key + self.name = name + self.object_id = object_id + self._get_sensitivity = get_sensitivity + self._set_sensitivity = set_sensitivity + self.value = initial_value + self._log = logging.getLogger(f"{self.__class__.__name__}[{self.key}]") + + def update_get_sensitivity(self, get_sensitivity: Callable[[], float]) -> None: + self._get_sensitivity = get_sensitivity + + def update_set_sensitivity(self, set_sensitivity: Callable[[float], None]) -> None: + self._set_sensitivity = set_sensitivity + + def sync_with_state(self) -> None: + old_value = self.value + self.value = self._get_sensitivity() + self._log.debug("Entity synchronized: old=%.3f new=%.3f", old_value, self.value) + + def handle_message(self, msg: message.Message) -> Iterable[message.Message]: + if isinstance(msg, NumberCommandRequest) and (msg.key == self.key): + new_value = float(msg.state) + self._log.debug("Second wake word sensitivity value changed: %s => %s", self.value, new_value) + self.value = new_value + self._set_sensitivity(new_value) + yield NumberStateResponse(key=self.key, state=self.value) + elif isinstance(msg, ListEntitiesRequest): + yield ListEntitiesNumberResponse( + object_id=self.object_id, + key=self.key, + name=self.name, + entity_category=EntityCategory.CONFIG, + min_value=0.0, + max_value=1.0, + step=0.001, + mode=NumberMode.BOX, + ) + elif isinstance(msg, SubscribeHomeAssistantStatesRequest): + self.sync_with_state() + yield NumberStateResponse(key=self.key, state=self.value) + + +class StopWordSensitivityNumberEntity(ESPHomeEntity): + def __init__( + self, + server: APIServer, + key: int, + name: str, + object_id: str, + get_sensitivity: Callable[[], float], + set_sensitivity: Callable[[float], None], + initial_value: float = 0.5, + ) -> None: + ESPHomeEntity.__init__(self, server) + + self.key = key + self.name = name + self.object_id = object_id + self._get_sensitivity = get_sensitivity + self._set_sensitivity = set_sensitivity + self.value = initial_value + self._log = logging.getLogger(f"{self.__class__.__name__}[{self.key}]") + + def update_get_sensitivity(self, get_sensitivity: Callable[[], float]) -> None: + self._get_sensitivity = get_sensitivity + + def update_set_sensitivity(self, set_sensitivity: Callable[[float], None]) -> None: + self._set_sensitivity = set_sensitivity + + def sync_with_state(self) -> None: + old_value = self.value + self.value = self._get_sensitivity() + self._log.debug("Entity synchronized: old=%.3f new=%.3f", old_value, self.value) + + def handle_message(self, msg: message.Message) -> Iterable[message.Message]: + if isinstance(msg, NumberCommandRequest) and (msg.key == self.key): + new_value = float(msg.state) + self._log.debug("Stop word sensitivity value changed: %s => %s", self.value, new_value) + self.value = new_value + self._set_sensitivity(new_value) + yield NumberStateResponse(key=self.key, state=self.value) + elif isinstance(msg, ListEntitiesRequest): + yield ListEntitiesNumberResponse( + object_id=self.object_id, + key=self.key, + name=self.name, + entity_category=EntityCategory.CONFIG, + icon="mdi:hand-back-left", + min_value=0.0, + max_value=1.0, + step=0.001, + mode=NumberMode.BOX, + ) + elif isinstance(msg, SubscribeHomeAssistantStatesRequest): + self.sync_with_state() + yield NumberStateResponse(key=self.key, state=self.value) + + +# Backward compatibility export aliases +__all__ = [ + "ESPHomeEntity", + "MediaPlayerEntity", + "MuteSwitchEntity", + "ThinkingSoundEntity", + "WakeWord1SensitivityNumberEntity", + "WakeWord2SensitivityNumberEntity", + "StopWordSensitivityNumberEntity", + # Old class names for backward compatibility + "WakeWordSensitivityNumberEntity", + "SecondWakeWordSensitivityNumberEntity", +] + +WakeWordSensitivityNumberEntity = WakeWord1SensitivityNumberEntity +SecondWakeWordSensitivityNumberEntity = WakeWord2SensitivityNumberEntity + + +# ----------------------------------------------------------------------------- diff --git a/linux_voice_assistant/models.py b/linux_voice_assistant/models.py index 65684f9..7319ee3 100644 --- a/linux_voice_assistant/models.py +++ b/linux_voice_assistant/models.py @@ -17,7 +17,10 @@ if TYPE_CHECKING: MediaPlayerEntity, MicSettingEntity, MuteSwitchEntity, + StopWordSensitivityNumberEntity, ThinkingSoundEntity, + WakeWord1SensitivityNumberEntity, + WakeWord2SensitivityNumberEntity, ) from .mpv_player import MpvMediaPlayer from .satellite import VoiceSatelliteProtocol @@ -37,6 +40,7 @@ class AvailableWakeWord: wake_word: str trained_languages: List[str] wake_word_path: Path + probability_cutoff: float = 0.7 def load(self) -> "Union[MicroWakeWord, OpenWakeWord]": if self.type == WakeWordType.MICRO_WAKE_WORD: @@ -57,9 +61,13 @@ class AvailableWakeWord: @dataclass class Preferences: - active_wake_words: List[str] = field(default_factory=list) + active_wake_words: List[Optional[str]] = field(default_factory=list) volume: Optional[float] = None thinking_sound: int = 0 # 0 = disabled, 1 = enabled + wake_word_1_sensitivity: Optional[float] = None + wake_word_2_sensitivity: Optional[float] = None + stop_word_sensitivity: Optional[float] = None + mic_auto_gain: int = 0 mic_noise_suppression: int = 0 mic_volume: int = 100 # 1–100, default maximum @@ -95,6 +103,9 @@ class ServerState: satellite: "Optional[VoiceSatelliteProtocol]" = None mute_switch_entity: "Optional[MuteSwitchEntity]" = None thinking_sound_entity: "Optional[ThinkingSoundEntity]" = None + sensitivity_1_number_entity: "Optional[WakeWord1SensitivityNumberEntity]" = None + sensitivity_2_number_entity: "Optional[WakeWord2SensitivityNumberEntity]" = None + stop_sensitivity_number_entity: "Optional[StopWordSensitivityNumberEntity]" = None mic_gain_entity: "Optional[MicSettingEntity]" = None mic_noise_suppression_entity: "Optional[MicSettingEntity]" = None mic_volume_entity: "Optional[MicSettingEntity]" = None @@ -105,6 +116,12 @@ class ServerState: muted: bool = False connected: bool = False volume: float = 1.0 + oww_probability_cutoff: float = 0.7 # Dynamic threshold for OpenWakeWord + oww_second_probability_cutoff: float = 0.7 # Dynamic threshold for second OpenWakeWord + oww_stop_probability_cutoff: float = 0.5 # Dynamic threshold for Stop word + wake_word_1_threshold: float = 0.7 + wake_word_2_threshold: float = 0.7 + stop_word_threshold: float = 0.5 mic_auto_gain: int = 0 mic_noise_suppression: int = 0 mic_volume: int = 100 # 1–100, default maximum diff --git a/linux_voice_assistant/satellite.py b/linux_voice_assistant/satellite.py index 1eeaf86..05184d0 100644 --- a/linux_voice_assistant/satellite.py +++ b/linux_voice_assistant/satellite.py @@ -47,7 +47,15 @@ from pymicro_wakeword import MicroWakeWord from pyopen_wakeword import OpenWakeWord from .api_server import APIServer -from .entity import MediaPlayerEntity, MicSettingEntity, MuteSwitchEntity, ThinkingSoundEntity +from .entity import ( + MediaPlayerEntity, + MicSettingEntity, + MuteSwitchEntity, + StopWordSensitivityNumberEntity, + ThinkingSoundEntity, + WakeWord1SensitivityNumberEntity, + WakeWord2SensitivityNumberEntity, +) from .models import AvailableWakeWord, ServerState, WakeWordType from .util import call_all @@ -161,6 +169,76 @@ class VoiceSatelliteProtocol(APIServer): thinking_sound_switch.update_set_thinking_sound_enabled(self._set_thinking_sound_enabled) thinking_sound_switch.sync_with_state() + # Add/update Wake Word 1 sensitivity number entity + sensitivity_1_entity = self.state.sensitivity_1_number_entity + if sensitivity_1_entity is None: + sensitivity_1_entity = WakeWord1SensitivityNumberEntity( + server=self, + key=len(state.entities), + name="Wake Word 1 Sensitivity", + object_id="wake_word_1_sensitivity", + get_sensitivity=lambda: self.state.wake_word_1_threshold, + set_sensitivity=self._set_sensitivity_1, + initial_value=self.state.wake_word_1_threshold, + ) + self.state.entities.append(sensitivity_1_entity) + self.state.sensitivity_1_number_entity = sensitivity_1_entity + elif sensitivity_1_entity not in self.state.entities: + self.state.entities.append(sensitivity_1_entity) + + sensitivity_1_entity.server = self + sensitivity_1_entity.update_get_sensitivity(lambda: self.state.wake_word_1_threshold) + sensitivity_1_entity.update_set_sensitivity(self._set_sensitivity_1) + + sensitivity_1_entity.sync_with_state() + _LOGGER.debug("INIT: Wake Word 1 entity initialized with value %.3f", sensitivity_1_entity.value) + + # Add/update Wake Word 2 sensitivity number entity + sensitivity_2_entity = self.state.sensitivity_2_number_entity + if sensitivity_2_entity is None: + sensitivity_2_entity = WakeWord2SensitivityNumberEntity( + server=self, + key=len(state.entities), + name="Wake Word 2 Sensitivity", + object_id="wake_word_2_sensitivity", + get_sensitivity=lambda: self.state.wake_word_2_threshold, + set_sensitivity=self._set_sensitivity_2, + initial_value=self.state.wake_word_2_threshold, + ) + self.state.entities.append(sensitivity_2_entity) + self.state.sensitivity_2_number_entity = sensitivity_2_entity + elif sensitivity_2_entity not in self.state.entities: + self.state.entities.append(sensitivity_2_entity) + + sensitivity_2_entity.server = self + sensitivity_2_entity.update_get_sensitivity(lambda: self.state.wake_word_2_threshold) + sensitivity_2_entity.update_set_sensitivity(self._set_sensitivity_2) + + sensitivity_2_entity.sync_with_state() + + # Add/update Stop Word sensitivity number entity + stop_sensitivity_entity = self.state.stop_sensitivity_number_entity + if stop_sensitivity_entity is None: + stop_sensitivity_entity = StopWordSensitivityNumberEntity( + server=self, + key=len(state.entities), + name="Stop Word Sensitivity", + object_id="stop_word_sensitivity", + get_sensitivity=lambda: self.state.stop_word_threshold, + set_sensitivity=self._set_stop_sensitivity, + initial_value=self.state.stop_word_threshold, + ) + self.state.entities.append(stop_sensitivity_entity) + self.state.stop_sensitivity_number_entity = stop_sensitivity_entity + elif stop_sensitivity_entity not in self.state.entities: + self.state.entities.append(stop_sensitivity_entity) + + stop_sensitivity_entity.server = self + stop_sensitivity_entity.update_get_sensitivity(lambda: self.state.stop_word_threshold) + stop_sensitivity_entity.update_set_sensitivity(self._set_stop_sensitivity) + + stop_sensitivity_entity.sync_with_state() + # Mic Gain if self.state.mic_gain_entity is None: self.state.mic_gain_entity = MicSettingEntity( @@ -258,6 +336,33 @@ class VoiceSatelliteProtocol(APIServer): pass self.state.save_preferences() + def _set_sensitivity_1(self, new_value: float) -> None: + self.state.wake_word_1_threshold = float(new_value) + self.state.preferences.wake_word_1_sensitivity = float(new_value) + self.state.save_preferences() + _LOGGER.debug("Wake Word 1 Sensitivity value set to: %s", new_value) + # Sync entity state + if self.state.sensitivity_1_number_entity is not None: + self.state.sensitivity_1_number_entity.sync_with_state() + + def _set_sensitivity_2(self, new_value: float) -> None: + self.state.wake_word_2_threshold = float(new_value) + self.state.preferences.wake_word_2_sensitivity = float(new_value) + self.state.save_preferences() + _LOGGER.debug("Wake Word 2 Sensitivity value set to: %s", new_value) + # Sync entity state + if self.state.sensitivity_2_number_entity is not None: + self.state.sensitivity_2_number_entity.sync_with_state() + + def _set_stop_sensitivity(self, new_value: float) -> None: + self.state.stop_word_threshold = float(new_value) + self.state.preferences.stop_word_sensitivity = float(new_value) + self.state.save_preferences() + _LOGGER.debug("Stop Word Sensitivity value set to: %s", new_value) + # Sync entity state + if self.state.stop_sensitivity_number_entity is not None: + self.state.stop_sensitivity_number_entity.sync_with_state() + def _set_muted(self, new_state: bool) -> None: self.state.muted = bool(new_state) @@ -387,6 +492,9 @@ class VoiceSatelliteProtocol(APIServer): if isinstance(msg, ListEntitiesRequest): yield ListEntitiesDoneResponse() elif isinstance(msg, VoiceAssistantConfigurationRequest): + _LOGGER.debug("✅ Received VoiceAssistantConfigurationRequest from Home Assistant") + _LOGGER.debug(" -> Request contains %d external wake words", len(msg.external_wake_words)) + available_wake_words = [ VoiceAssistantWakeWord( id=ww.id, @@ -396,10 +504,20 @@ class VoiceSatelliteProtocol(APIServer): for ww in self.state.available_wake_words.values() ] + # Log available internal wake words first + internal_ww_count = len(self.state.available_wake_words) + _LOGGER.debug(" -> Found %d internal available wake words", internal_ww_count) + for ww in available_wake_words: + _LOGGER.debug(" - %s: '%s' (langs: %s)", ww.id, ww.wake_word, ww.trained_languages) + for eww in msg.external_wake_words: + _LOGGER.debug(" -> Processing external wake word: id=%s, word='%s', type=%s", eww.id, eww.wake_word, eww.model_type) + if eww.model_type != "micro": + _LOGGER.debug(" → Skipping: not micro model type") continue + _LOGGER.debug(" → Adding to available wake words") available_wake_words.append( VoiceAssistantWakeWord( id=eww.id, @@ -409,47 +527,83 @@ class VoiceSatelliteProtocol(APIServer): ) self._external_wake_words[eww.id] = eww + _LOGGER.debug(" → Stored in external wake words cache") + + active_ww_ids = [ww.id for ww in self.state.wake_words.values() if ww.id in self.state.active_wake_words] + _LOGGER.debug(" -> Active wake word IDs: %s", active_ww_ids) yield VoiceAssistantConfigurationResponse( available_wake_words=available_wake_words, - active_wake_words=[ww.id for ww in self.state.wake_words.values() if ww.id in self.state.active_wake_words], + active_wake_words=active_ww_ids, max_active_wake_words=2, ) - _LOGGER.info("Connected to Home Assistant") + + _LOGGER.info("✅ Connected to Home Assistant - Configuration handshake completed") + _LOGGER.debug("✅ VoiceAssistantConfigurationResponse sent successfully") elif isinstance(msg, VoiceAssistantSetConfiguration): # Change active wake words active_wake_words: Set[str] = set() + new_wake_words: List[Optional[str]] = [None, None] + # Get old positions before modification + old_positions: Dict[str, int] = {} + for idx, ww_id in enumerate(self.state.preferences.active_wake_words): + if ww_id is not None and idx < 2: + old_positions[ww_id] = idx + + # Process new active wake words for wake_word_id in msg.active_wake_words: if wake_word_id in self.state.wake_words: # Already active active_wake_words.add(wake_word_id) - continue - - model_info = self.state.available_wake_words.get(wake_word_id) - if not model_info: - # Check external wake words (may require download) - external_wake_word = self._external_wake_words.get(wake_word_id) - if not external_wake_word: - continue - - model_info = self._download_external_wake_word(external_wake_word) + else: + model_info = self.state.available_wake_words.get(wake_word_id) if not model_info: - continue + # Check external wake words (may require download) + external_wake_word = self._external_wake_words.get(wake_word_id) + if not external_wake_word: + continue - self.state.available_wake_words[wake_word_id] = model_info + model_info = self._download_external_wake_word(external_wake_word) + if not model_info: + continue - _LOGGER.debug("Loading wake word: %s", model_info.wake_word_path) - self.state.wake_words[wake_word_id] = model_info.load() + self.state.available_wake_words[wake_word_id] = model_info - _LOGGER.info("Wake word set: %s", wake_word_id) - active_wake_words.add(wake_word_id) - break + _LOGGER.debug("Loading wake word: %s", model_info.wake_word_path) + self.state.wake_words[wake_word_id] = model_info.load() + + _LOGGER.info("Wake word set: %s", wake_word_id) + active_wake_words.add(wake_word_id) + + # Keep old positions + remaining_ww = list(active_wake_words) + placed = set() + + # First, place Wake Words in their old positions. + for ww_id in remaining_ww: + if ww_id in old_positions: + pos = old_positions[ww_id] + if pos < 2: + new_wake_words[pos] = ww_id + placed.add(ww_id) + + # Add remaining wake words to free slots + free_slots = [i for i in range(2) if new_wake_words[i] is None] + for ww_id in remaining_ww: + if ww_id not in placed and free_slots: + pos = free_slots.pop(0) + new_wake_words[pos] = ww_id + placed.add(ww_id) + + # If only one wake word is left and it was at position 1, position 0 remains None + # Position 2 automatically stays None if not occupied self.state.active_wake_words = active_wake_words _LOGGER.debug("Active wake words: %s", active_wake_words) + _LOGGER.debug("Wake word positions: [0]=%s, [1]=%s", new_wake_words[0], new_wake_words[1]) - self.state.preferences.active_wake_words = list(active_wake_words) + self.state.preferences.active_wake_words = new_wake_words self.state.save_preferences() self.state.wake_words_changed = True diff --git a/linux_voice_assistant/wake_word.py b/linux_voice_assistant/wake_word.py new file mode 100644 index 0000000..d5a3180 --- /dev/null +++ b/linux_voice_assistant/wake_word.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +import json +import logging +from pathlib import Path +from typing import Dict, List, Optional, Set, Union + +from pymicro_wakeword import MicroWakeWord +from pyopen_wakeword import OpenWakeWord + +from .models import AvailableWakeWord, WakeWordType + +_LOGGER = logging.getLogger(__name__) + + +def find_available_wake_words(wake_word_dirs: List[Path], stop_model_id: str) -> Dict[str, AvailableWakeWord]: + """ + Searches all available wake words in the specified directories. + Loads configurations and creates AvailableWakeWord objects. + + Args: + wake_word_dirs: List of directories to search for wake words + stop_model_id: ID of the stop model which will not be listed as available wake word + + Returns: + Dictionary with wake word ID as key and AvailableWakeWord object as value + """ + available_wake_words: Dict[str, AvailableWakeWord] = {} + + _LOGGER.debug("Searching for wake words in directories: %s", [str(d) for d in wake_word_dirs]) + + for wake_word_dir in wake_word_dirs: + _LOGGER.debug("Checking directory: %s (exists: %s)", wake_word_dir, wake_word_dir.exists()) + + config_files = list(wake_word_dir.glob("*.json")) + _LOGGER.debug("Found %d JSON configuration files in %s", len(config_files), wake_word_dir) + + for model_config_path in config_files: + _LOGGER.debug("Processing configuration file: %s", model_config_path) + + model_id = model_config_path.stem + if model_id == stop_model_id: + # Skip stop model, do not show as available wake word + _LOGGER.debug("Skipping stop model: %s", model_id) + continue + + with open(model_config_path, "r", encoding="utf-8") as model_config_file: + model_config = json.load(model_config_file) + model_type = WakeWordType(model_config["type"]) + + _LOGGER.debug("Model %s is of type: %s", model_id, model_type) + + if model_type == WakeWordType.OPEN_WAKE_WORD: + wake_word_path = model_config_path.parent / model_config["model"] + else: + wake_word_path = model_config_path + + _LOGGER.debug("Model path resolved to: %s (exists: %s)", wake_word_path, wake_word_path.exists()) + + # Get type specific configuration + type_config = model_config.get(model_type.value, {}) + + _LOGGER.debug("Type specific config for %s (%s): %s", model_id, model_type.value, type_config) + + available_wake_words[model_id] = AvailableWakeWord( + id=model_id, + type=WakeWordType(model_type), + wake_word=model_config["wake_word"], + trained_languages=model_config.get("trained_languages", []), + wake_word_path=wake_word_path, + probability_cutoff=type_config.get("probability_cutoff", 0.7), + ) + _LOGGER.debug("Successfully registered wake word: %s", model_id) + + _LOGGER.debug("Total available wake words found: %d", len(available_wake_words)) + _LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys()))) + return available_wake_words + + +def load_wake_models( + available_wake_words: Dict[str, AvailableWakeWord], active_wake_word_ids: Optional[List[str]], default_wake_word_id: str +) -> tuple[Dict[str, Union[MicroWakeWord, OpenWakeWord]], Set[str], bool]: + """ + Loads the specified wake word models. + + If no active wake words are provided, the default model will be loaded. + + Args: + available_wake_words: Dictionary with all available wake words + active_wake_word_ids: List of IDs of wake words to load (may be None) + default_wake_word_id: ID of the default model which is loaded if no others are specified + + Returns: + Tuple with (Dictionary of loaded models, Set of active wake word IDs) + """ + active_wake_words: Set[str] = set() + wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {} + + _LOGGER.debug("Requested active wake word ids: %s", active_wake_word_ids) + _LOGGER.debug("Default wake word id: %s", default_wake_word_id) + + if active_wake_word_ids: + # Load preferred models + _LOGGER.debug("Loading requested wake word models, count: %d", len(active_wake_word_ids)) + for index, wake_word_id in enumerate(active_wake_word_ids): + _LOGGER.debug("Processing wake word %d/%d: %s", index + 1, len(active_wake_word_ids), wake_word_id) + wake_word = available_wake_words.get(wake_word_id) + if wake_word is None: + _LOGGER.warning("Unknown wake word ID: %s - skipping", wake_word_id) + continue + + _LOGGER.debug("Loading wake model: %s (%s)", wake_word_id, wake_word.wake_word) + try: + wake_models[wake_word_id] = wake_word.load() + active_wake_words.add(wake_word_id) + _LOGGER.debug("✅ Successfully loaded wake model: %s", wake_word_id) + except Exception as ex: + _LOGGER.error("❌ Failed to load wake model %s: %s", wake_word_id, ex, exc_info=True) + + if not wake_models: + # No models loaded, fall back to default model + _LOGGER.debug("No wake models loaded, falling back to default model") + wake_word_id = default_wake_word_id + + # Check if default wake word exists + wake_word = available_wake_words.get(wake_word_id) + if wake_word is None: + _LOGGER.error("❌ Default wake word '%s' not found!", wake_word_id) + + # Try fallback to 'okay_nabu' + wake_word_id = "okay_nabu" + wake_word = available_wake_words.get(wake_word_id) + + if wake_word is None: + _LOGGER.error("❌ Fallback wake word 'okay_nabu' also not found!") + + # If absolutely nothing works, take first available wake word + if available_wake_words: + wake_word_id = next(iter(available_wake_words.keys())) + wake_word = available_wake_words[wake_word_id] + _LOGGER.warning("⚠️ Using first available wake word as last resort: %s", wake_word_id) + else: + _LOGGER.critical("❌ NO WAKE WORDS FOUND AT ALL! Cannot proceed.") + raise RuntimeError("No wake word models available in any search directory") + + # wake_word_id2 = "hey_home_assistant" + # wake_word2 = available_wake_words.get(wake_word_id2) + + _LOGGER.debug("Loading default wake model 1: %s", wake_word_id) + # _LOGGER.debug("Loading default wake model 2: %s", wake_word_id2) + try: + wake_models[wake_word_id] = wake_word.load() + # if wake_word2 is not None: + # wake_models[wake_word_id2] = wake_word2.load() + # active_wake_words.add(wake_word_id2) + + active_wake_words.add(wake_word_id) + _LOGGER.debug("✅ Successfully loaded default wake model 1: %s", wake_word_id) + # _LOGGER.debug("✅ Successfully loaded default wake model 2: %s", wake_word_id2) + except Exception as ex: + _LOGGER.critical("❌ Failed to load even fallback wake word %s: %s", wake_word_id, ex, exc_info=True) + raise + + _LOGGER.debug("Loaded %d wake models successfully", len(wake_models)) + _LOGGER.debug("Active wake words: %s", sorted(active_wake_words)) + + fallback_used = not active_wake_word_ids or not wake_models + + return wake_models, active_wake_words, fallback_used + + +def load_stop_model(wake_word_dirs: List[Path], stop_model_id: str) -> Optional[MicroWakeWord]: + """ + Loads the stop word model. + + Args: + wake_word_dirs: List of directories to search for the stop model + stop_model_id: ID of the stop model + + Returns: + Loaded MicroWakeWord object or None if not found + """ + _LOGGER.debug("Searching for stop model '%s' in directories: %s", stop_model_id, [str(d) for d in wake_word_dirs]) + + for wake_word_dir in wake_word_dirs: + stop_config_path = wake_word_dir / f"{stop_model_id}.json" + _LOGGER.debug("Checking stop model path: %s (exists: %s)", stop_config_path, stop_config_path.exists()) + + if not stop_config_path.exists(): + continue + + _LOGGER.debug("Found stop model configuration at: %s", stop_config_path) + _LOGGER.debug("Loading stop model: %s", stop_config_path) + + try: + model = MicroWakeWord.from_config(stop_config_path) + _LOGGER.debug("Successfully loaded stop model") + return model + except Exception as ex: + _LOGGER.error("Failed to load stop model from %s: %s", stop_config_path, ex, exc_info=True) + + _LOGGER.warning("Stop model '%s' could not be found in any search directory", stop_model_id) + return None diff --git a/mypy.ini b/mypy.ini index 497e569..cebf0f6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,3 +9,8 @@ ignore_missing_imports = True [mypy-soundcard.*] ignore_missing_imports = True + +[mypy-aioesphomeapi.api_pb2] +ignore_missing_imports = True +allow_subclassing_any = True +disable_error_code = attr-defined diff --git a/wakewords/openWakeWord/alexa_v0.1.json b/wakewords/openWakeWord/alexa_v0.1.json index c7a4d2e..64d5981 100644 --- a/wakewords/openWakeWord/alexa_v0.1.json +++ b/wakewords/openWakeWord/alexa_v0.1.json @@ -1,5 +1,8 @@ { "type": "openWakeWord", "wake_word": "Alexa", - "model": "alexa_v0.1.tflite" + "model": "alexa_v0.1.tflite", + "openWakeWord": { + "probability_cutoff": 0.7 + } } diff --git a/wakewords/openWakeWord/hey_jarvis_v0.1.json b/wakewords/openWakeWord/hey_jarvis_v0.1.json index b7d632e..85c68f8 100644 --- a/wakewords/openWakeWord/hey_jarvis_v0.1.json +++ b/wakewords/openWakeWord/hey_jarvis_v0.1.json @@ -1,5 +1,8 @@ { "type": "openWakeWord", "wake_word": "Hey Jarvis", - "model": "hey_jarvis_v0.1.tflite" + "model": "hey_jarvis_v0.1.tflite", + "openWakeWord": { + "probability_cutoff": 0.7 + } } diff --git a/wakewords/openWakeWord/hey_mycroft_v0.1.json b/wakewords/openWakeWord/hey_mycroft_v0.1.json index abd5118..311e447 100644 --- a/wakewords/openWakeWord/hey_mycroft_v0.1.json +++ b/wakewords/openWakeWord/hey_mycroft_v0.1.json @@ -1,5 +1,8 @@ { "type": "openWakeWord", "wake_word": "Hey Mycroft", - "model": "hey_mycroft_v0.1.tflite" + "model": "hey_mycroft_v0.1.tflite", + "openWakeWord": { + "probability_cutoff": 0.7 + } } diff --git a/wakewords/openWakeWord/hey_rhasspy_v0.1.json b/wakewords/openWakeWord/hey_rhasspy_v0.1.json index 1ecc454..98ee38a 100644 --- a/wakewords/openWakeWord/hey_rhasspy_v0.1.json +++ b/wakewords/openWakeWord/hey_rhasspy_v0.1.json @@ -1,5 +1,8 @@ { "type": "openWakeWord", "wake_word": "Hey Rhasspy", - "model": "hey_rhasspy_v0.1.tflite" + "model": "hey_rhasspy_v0.1.tflite", + "openWakeWord": { + "probability_cutoff": 0.7 + } } diff --git a/wakewords/openWakeWord/ok_nabu_v0.1.json b/wakewords/openWakeWord/ok_nabu_v0.1.json index 3fae45e..dad14da 100644 --- a/wakewords/openWakeWord/ok_nabu_v0.1.json +++ b/wakewords/openWakeWord/ok_nabu_v0.1.json @@ -1,5 +1,8 @@ { "type": "openWakeWord", - "wake_word": "Okay Nabu", - "model": "ok_nabu_v0.1.tflite" + "wake_word": "Ok Nabu", + "model": "ok_nabu_v0.1.tflite", + "openWakeWord": { + "probability_cutoff": 0.7 + } }