Use pymicro-wakeword and pyopen-wakeword libraries

lib/SOURCE

@@ -1 +0,0 @@
-https://github.com/tphakala/tflite_c/

linux_voice_assistant/__main__.py

@@ -7,15 +7,15 @@ import threading
 import time
 from pathlib import Path
 from queue import Queue
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Set, Union
 
 import numpy as np
 import sounddevice as sd
+from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures
+from pyopen_wakeword import OpenWakeWord, OpenWakeWordFeatures
 
-from .microwakeword import MicroWakeWord, MicroWakeWordFeatures
 from .models import AvailableWakeWord, Preferences, ServerState, WakeWordType
 from .mpv_player import MpvMediaPlayer
-from .openwakeword import OpenWakeWord, OpenWakeWordFeatures
 from .satellite import VoiceSatelliteProtocol
 from .util import get_mac, is_arm
 from .zeroconf import HomeAssistantZeroconf
@@ -63,17 +63,6 @@ async def main() -> None:
         help="Seconds before wake word can be activated again",
     )
     #
-    parser.add_argument(
-        "--oww-melspectrogram-model",
-        default=_OWW_DIR / "melspectrogram.tflite",
-        help="Path to openWakeWord melspectrogram model",
-    )
-    parser.add_argument(
-        "--oww-embedding-model",
-        default=_OWW_DIR / "embedding_model.tflite",
-        help="Path to openWakeWord embedding model",
-    )
-    #
     parser.add_argument(
         "--wakeup-sound", default=str(_SOUNDS_DIR / "wake_word_triggered.flac")
     )
@@ -117,13 +106,18 @@ async def main() -> None:
 
             with open(model_config_path, "r", encoding="utf-8") as model_config_file:
                 model_config = json.load(model_config_file)
-                model_type = model_config["type"]
+                model_type = WakeWordType(model_config["type"])
+                if model_type == WakeWordType.OPEN_WAKE_WORD:
+                    wake_word_path = model_config_path.parent / model_config["model"]
+                else:
+                    wake_word_path = model_config_path
+
                 available_wake_words[model_id] = AvailableWakeWord(
                     id=model_id,
                     type=WakeWordType(model_type),
                     wake_word=model_config["wake_word"],
                     trained_languages=model_config.get("trained_languages", []),
-                    config_path=model_config_path,
+                    wake_word_path=wake_word_path,
                 )
 
     _LOGGER.debug("Available wake words: %s", list(sorted(available_wake_words.keys())))
@@ -138,10 +132,8 @@ async def main() -> None:
     else:
         preferences = Preferences()
 
-    libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so"
-    _LOGGER.debug("libtensorflowlite_c path: %s", libtensorflowlite_c_path)
-
     # Load wake/stop models
+    active_wake_words: Set[str] = set()
     wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {}
     if preferences.active_wake_words:
         # Load preferred models
@@ -152,7 +144,8 @@ async def main() -> None:
                 continue
 
             _LOGGER.debug("Loading wake model: %s", wake_word_id)
-            wake_models[wake_word_id] = wake_word.load(libtensorflowlite_c_path)
+            wake_models[wake_word_id] = wake_word.load()
+            active_wake_words.add(wake_word_id)
 
     if not wake_models:
         # Load default model
@@ -160,7 +153,8 @@ async def main() -> None:
         wake_word = available_wake_words[wake_word_id]
 
         _LOGGER.debug("Loading wake model: %s", wake_word_id)
-        wake_models[wake_word_id] = wake_word.load(libtensorflowlite_c_path)
+        wake_models[wake_word_id] = wake_word.load()
+        active_wake_words.add(wake_word_id)
 
     # TODO: allow openWakeWord for "stop"
     stop_model: Optional[MicroWakeWord] = None
@@ -170,9 +164,7 @@ async def main() -> None:
             continue
 
         _LOGGER.debug("Loading stop model: %s", stop_config_path)
-        stop_model = MicroWakeWord.from_config(
+        stop_model = MicroWakeWord.from_config(stop_config_path)
-            stop_config_path, libtensorflowlite_c_path
-        )
         break
 
     assert stop_model is not None
@@ -184,6 +176,7 @@ async def main() -> None:
         entities=[],
         available_wake_words=available_wake_words,
         wake_words=wake_models,
+        active_wake_words=active_wake_words,
         stop_word=stop_model,
         music_player=MpvMediaPlayer(device=args.audio_output_device),
         tts_player=MpvMediaPlayer(device=args.audio_output_device),
@@ -191,9 +184,6 @@ async def main() -> None:
         timer_finished_sound=args.timer_finished_sound,
         preferences=preferences,
         preferences_path=preferences_path,
-        libtensorflowlite_c_path=libtensorflowlite_c_path,
-        oww_melspectrogram_path=Path(args.oww_melspectrogram_model),
-        oww_embedding_path=Path(args.oww_embedding_model),
         refractory_seconds=args.refractory_seconds,
     )
 
@@ -264,7 +254,11 @@ def process_audio(state: ServerState):
             if (not wake_words) or (state.wake_words_changed and state.wake_words):
                 # Update list of wake word models to process
                 state.wake_words_changed = False
-                wake_words = [ww for ww in state.wake_words.values() if ww.is_active]
+                wake_words = [
+                    ww
+                    for ww in state.wake_words.values()
+                    if ww.id in state.active_wake_words
+                ]
 
                 has_oww = False
                 for wake_word in wake_words:
@@ -272,16 +266,10 @@ def process_audio(state: ServerState):
                         has_oww = True
 
                 if micro_features is None:
-                    micro_features = MicroWakeWordFeatures(
+                    micro_features = MicroWakeWordFeatures()
-                        libtensorflowlite_c_path=state.libtensorflowlite_c_path,
-                    )
 
                 if has_oww and (oww_features is None):
-                    oww_features = OpenWakeWordFeatures(
+                    oww_features = OpenWakeWordFeatures.from_builtin()
-                        melspectrogram_model=state.oww_melspectrogram_path,
-                        embedding_model=state.oww_embedding_path,
-                        libtensorflowlite_c_path=state.libtensorflowlite_c_path,
-                    )
 
             try:
                 state.satellite.handle_audio(audio_chunk)
@@ -322,7 +310,7 @@ def process_audio(state: ServerState):
                     if state.stop_word.process_streaming(micro_input):
                         stopped = True
 
-                if stopped and state.stop_word.is_active:
+                if stopped and (state.stop_word.id in state.active_wake_words):
                     state.satellite.stop()
             except Exception:
                 _LOGGER.exception("Unexpected error handling audio")

linux_voice_assistant/microwakeword.py

@@ -1,189 +0,0 @@
-import ctypes
-import json
-import statistics
-from collections import deque
-from collections.abc import Iterable
-from pathlib import Path
-from typing import Deque, List, Union
-
-import numpy as np
-from pymicro_features import MicroFrontend
-
-from .wakeword import TfLiteWakeWord
-
-SAMPLES_PER_SECOND = 16000
-SAMPLES_PER_CHUNK = 160  # 10ms
-BYTES_PER_SAMPLE = 2  # 16-bit
-BYTES_PER_CHUNK = SAMPLES_PER_CHUNK * BYTES_PER_SAMPLE
-SECONDS_PER_CHUNK = SAMPLES_PER_CHUNK / SAMPLES_PER_SECOND
-STRIDE = 3
-
-
-class MicroWakeWord(TfLiteWakeWord):
-    def __init__(
-        self,
-        id: str,  # pylint: disable=redefined-builtin
-        wake_word: str,
-        tflite_model: Union[str, Path],
-        probability_cutoff: float,
-        sliding_window_size: int,
-        trained_languages: List[str],
-        libtensorflowlite_c_path: Union[str, Path],
-    ) -> None:
-        TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
-
-        self.id = id
-        self.wake_word = wake_word
-        self.tflite_model = tflite_model
-        self.probability_cutoff = probability_cutoff
-        self.sliding_window_size = sliding_window_size
-        self.trained_languages = trained_languages
-
-        self.is_active = True
-
-        # Load the model and create interpreter
-        self.model_path = str(Path(tflite_model).resolve()).encode("utf-8")
-        self._load_model()
-
-        self._features: List[np.ndarray] = []
-        self._probabilities: Deque[float] = deque(maxlen=self.sliding_window_size)
-        self._audio_buffer = bytes()
-
-    def _load_model(self) -> None:
-        self.model = self.lib.TfLiteModelCreateFromFile(self.model_path)
-        self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None)
-        self.lib.TfLiteInterpreterAllocateTensors(self.interpreter)
-
-        # Access input and output tensor
-        self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
-            self.interpreter, 0
-        )
-        self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
-            self.interpreter, 0
-        )
-
-        # Get quantization parameters
-        input_q = self.lib.TfLiteTensorQuantizationParams(self.input_tensor)
-        output_q = self.lib.TfLiteTensorQuantizationParams(self.output_tensor)
-
-        self.input_scale, self.input_zero_point = input_q.scale, input_q.zero_point
-        self.output_scale, self.output_zero_point = output_q.scale, output_q.zero_point
-
-    @staticmethod
-    def from_config(
-        config_path: Union[str, Path],
-        libtensorflowlite_c_path: Union[str, Path],
-    ) -> "MicroWakeWord":
-        """Load a microWakeWord model from a JSON config file.
-
-        Parameters
-        ----------
-        config_path: str or Path
-            Path to JSON configuration file
-        """
-        config_path = Path(config_path)
-        with open(config_path, "r", encoding="utf-8") as config_file:
-            config = json.load(config_file)
-
-        micro_config = config["micro"]
-
-        return MicroWakeWord(
-            id=Path(config["model"]).stem,
-            wake_word=config["wake_word"],
-            tflite_model=config_path.parent / config["model"],
-            probability_cutoff=micro_config["probability_cutoff"],
-            sliding_window_size=micro_config["sliding_window_size"],
-            trained_languages=micro_config.get("trained_languages", []),
-            libtensorflowlite_c_path=libtensorflowlite_c_path,
-        )
-
-    def process_streaming(self, features: np.ndarray) -> bool:
-        self._features.append(features)
-
-        if len(self._features) < STRIDE:
-            # Not enough windows
-            return False
-
-        # Allocate and quantize input data
-        quant_features = np.round(
-            np.concatenate(self._features, axis=1) / self.input_scale
-            + self.input_zero_point
-        ).astype(np.uint8)
-
-        # Stride instead of rolling
-        self._features.clear()
-
-        # Set tensor
-        quant_ptr = quant_features.ctypes.data_as(ctypes.c_void_p)
-        self.lib.TfLiteTensorCopyFromBuffer(
-            self.input_tensor, quant_ptr, quant_features.nbytes
-        )
-
-        # Run inference
-        self.lib.TfLiteInterpreterInvoke(self.interpreter)
-
-        # Read output
-        output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor)
-        output_data = np.empty(output_bytes, dtype=np.uint8)
-        self.lib.TfLiteTensorCopyToBuffer(
-            self.output_tensor,
-            output_data.ctypes.data_as(ctypes.c_void_p),
-            output_bytes,
-        )
-
-        # Dequantize output
-        result = (
-            output_data.astype(np.float32) - self.output_zero_point
-        ) * self.output_scale
-
-        self._probabilities.append(result.item())
-
-        if len(self._probabilities) < self.sliding_window_size:
-            # Not enough probabilities
-            return False
-
-        if statistics.mean(self._probabilities) > self.probability_cutoff:
-            return True
-
-        return False
-
-
-# -----------------------------------------------------------------------------
-
-
-class MicroWakeWordFeatures(TfLiteWakeWord):
-    def __init__(
-        self,
-        libtensorflowlite_c_path: Union[str, Path],
-    ) -> None:
-        TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
-
-        self._audio_buffer = bytes()
-        self._frontend = MicroFrontend()
-
-    def process_streaming(self, audio_bytes: bytes) -> Iterable[np.ndarray]:
-        self._audio_buffer += audio_bytes
-
-        if len(self._audio_buffer) < BYTES_PER_CHUNK:
-            # Not enough audio to get features
-            return
-
-        audio_buffer_idx = 0
-        while (audio_buffer_idx + BYTES_PER_CHUNK) <= len(self._audio_buffer):
-            # Process chunk
-            chunk_bytes = self._audio_buffer[
-                audio_buffer_idx : audio_buffer_idx + BYTES_PER_CHUNK
-            ]
-            frontend_result = self._frontend.ProcessSamples(chunk_bytes)
-            audio_buffer_idx += frontend_result.samples_read * BYTES_PER_SAMPLE
-
-            if not frontend_result.features:
-                # Not enough audio for a full window
-                continue
-
-            yield np.array(frontend_result.features).reshape(
-                (1, 1, len(frontend_result.features))
-            )
-
-        # Remove processed audio
-        self._audio_buffer = self._audio_buffer[audio_buffer_idx:]

linux_voice_assistant/models.py

@@ -6,13 +6,14 @@ from dataclasses import asdict, dataclass, field
 from enum import Enum
 from pathlib import Path
 from queue import Queue
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Union
 
 if TYPE_CHECKING:
+    from pymicro_wakeword import MicroWakeWord
+    from pyopen_wakeword import OpenWakeWord
+
     from .entity import ESPHomeEntity, MediaPlayerEntity
-    from .microwakeword import MicroWakeWord
     from .mpv_player import MpvMediaPlayer
-    from .openwakeword import OpenWakeWord
     from .satellite import VoiceSatelliteProtocol
 
 _LOGGER = logging.getLogger(__name__)
@@ -29,26 +30,21 @@ class AvailableWakeWord:
     type: WakeWordType
     wake_word: str
     trained_languages: List[str]
-    config_path: Path
+    wake_word_path: Path
 
-    def load(
+    def load(self) -> "Union[MicroWakeWord, OpenWakeWord]":
-        self, libtensorflowlite_c_path: Path
-    ) -> "Union[MicroWakeWord, OpenWakeWord]":
         if self.type == WakeWordType.MICRO_WAKE_WORD:
-            from .microwakeword import MicroWakeWord
+            from pymicro_wakeword import MicroWakeWord
 
-            return MicroWakeWord.from_config(
+            return MicroWakeWord.from_config(config_path=self.wake_word_path)
-                config_path=self.config_path,
-                libtensorflowlite_c_path=libtensorflowlite_c_path,
-            )
 
         if self.type == WakeWordType.OPEN_WAKE_WORD:
-            from .openwakeword import OpenWakeWord
+            from pyopen_wakeword import OpenWakeWord
 
-            return OpenWakeWord.from_config(
+            oww_model = OpenWakeWord.from_model(model_path=self.wake_word_path)
-                config_path=self.config_path,
+            setattr(oww_model, "wake_word", self.wake_word)
-                libtensorflowlite_c_path=libtensorflowlite_c_path,
+
-            )
+            return oww_model
 
         raise ValueError(f"Unexpected wake word type: {self.type}")
 
@@ -66,6 +62,7 @@ class ServerState:
     entities: "List[ESPHomeEntity]"
     available_wake_words: "Dict[str, AvailableWakeWord]"
     wake_words: "Dict[str, Union[MicroWakeWord, OpenWakeWord]]"
+    active_wake_words: Set[str]
     stop_word: "MicroWakeWord"
     music_player: "MpvMediaPlayer"
     tts_player: "MpvMediaPlayer"
@@ -73,11 +70,6 @@ class ServerState:
     timer_finished_sound: str
     preferences: Preferences
     preferences_path: Path
-    libtensorflowlite_c_path: Path
-
-    # openWakeWord
-    oww_melspectrogram_path: Path
-    oww_embedding_path: Path
 
     media_player_entity: "Optional[MediaPlayerEntity]" = None
     satellite: "Optional[VoiceSatelliteProtocol]" = None

linux_voice_assistant/openwakeword.py

@@ -1,293 +0,0 @@
-"""openWakeWord implementation."""
-
-import ctypes as C
-import json
-from collections.abc import Iterable
-from pathlib import Path
-from typing import Final, Union
-
-import numpy as np
-
-from .wakeword import TfLiteWakeWord
-
-c_void_p = C.c_void_p
-c_int32 = C.c_int32
-c_size_t = C.c_size_t
-
-BATCH_SIZE: Final = 1
-
-AUTOFILL_SECONDS: Final = 8
-MAX_SECONDS: Final = 10
-
-SAMPLE_RATE: Final = 16000  # 16Khz
-_MAX_SAMPLES: Final = MAX_SECONDS * SAMPLE_RATE
-
-SAMPLES_PER_CHUNK: Final = 1280  # 80 ms @ 16Khz
-MS_PER_CHUNK: Final = SAMPLES_PER_CHUNK // SAMPLE_RATE
-
-# window = 400, hop length = 160
-MELS_PER_SECOND: Final = 97
-MAX_MELS: Final = MAX_SECONDS * MELS_PER_SECOND
-MEL_SAMPLES: Final = 1760
-NUM_MELS: Final = 32
-
-EMB_FEATURES: Final = 76  # 775 ms
-EMB_STEP: Final = 8
-MAX_EMB: Final = MAX_SECONDS * EMB_STEP
-WW_FEATURES: Final = 96
-
-MEL_SHAPE: Final = (BATCH_SIZE, MEL_SAMPLES)
-EMB_SHAPE: Final = (BATCH_SIZE, EMB_FEATURES, NUM_MELS, 1)
-
-# melspec = [batch x samples (min: 1280)] => [batch x 1 x window x mels (32)]
-# stft window size: 25ms (400)
-# stft window step: 10ms (160)
-# mel band limits: 60Hz - 3800Hz
-# mel frequency bins: 32
-#
-# embedding = [batch x window x mels (32) x 1] => [batch x 1 x 1 x features (96)]
-# ww = [batch x window x features (96)] => [batch x probability]
-
-
-class OpenWakeWord(TfLiteWakeWord):
-    def __init__(
-        self,
-        id: str,  # pylint: disable=redefined-builtin
-        wake_word: str,
-        tflite_model: Union[str, Path],
-        libtensorflowlite_c_path: Union[str, Path],
-    ):
-        TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
-
-        self.id = id
-        self.wake_word = wake_word
-        self.tflite_model = tflite_model
-
-        self.is_active = True
-
-        # Load the model and create interpreter
-        self.model_path = str(Path(tflite_model).resolve()).encode("utf-8")
-        self.model = self.lib.TfLiteModelCreateFromFile(self.model_path)
-        self.interpreter = self.lib.TfLiteInterpreterCreate(self.model, None)
-        self.lib.TfLiteInterpreterAllocateTensors(self.interpreter)
-
-        self.input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
-            self.interpreter, c_int32(0)
-        )
-        self.output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
-            self.interpreter, c_int32(0)
-        )
-
-        num_input_dims = self.lib.TfLiteTensorNumDims(self.input_tensor)
-        input_shape = [
-            self.lib.TfLiteTensorDim(self.input_tensor, i)
-            for i in range(num_input_dims)
-        ]
-        self.input_windows = input_shape[1]
-
-        self.new_embeddings: int = 0
-        self.embeddings: np.ndarray = np.zeros(
-            shape=(MAX_EMB, WW_FEATURES), dtype=np.float32
-        )
-
-    def process_streaming(self, embeddings: np.ndarray) -> Iterable[float]:
-        """Generate probabilities from embeddings."""
-        num_embedding_windows = embeddings.shape[2]
-
-        # Shift
-        self.embeddings[:-num_embedding_windows] = self.embeddings[
-            num_embedding_windows:
-        ]
-
-        # Overwrite
-        self.embeddings[-num_embedding_windows:] = embeddings[0, 0, :, :]
-        self.new_embeddings = min(
-            len(self.embeddings),
-            self.new_embeddings + num_embedding_windows,
-        )
-
-        while self.new_embeddings >= self.input_windows:
-            emb_tensor = np.zeros(
-                shape=(1, self.input_windows, WW_FEATURES),
-                dtype=np.float32,
-            )
-
-            emb_tensor[0, :] = self.embeddings[
-                -self.new_embeddings : len(self.embeddings)
-                - self.new_embeddings
-                + self.input_windows
-            ]
-            self.new_embeddings = max(0, self.new_embeddings - 1)
-
-            # Run inference
-            emb_ptr = emb_tensor.ctypes.data_as(c_void_p)
-            self.lib.TfLiteTensorCopyFromBuffer(
-                self.input_tensor, emb_ptr, c_size_t(emb_tensor.nbytes)
-            )
-            self.lib.TfLiteInterpreterInvoke(self.interpreter)
-
-            output_bytes = self.lib.TfLiteTensorByteSize(self.output_tensor)
-            probs = np.empty(
-                output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32
-            )
-            self.lib.TfLiteTensorCopyToBuffer(
-                self.output_tensor,
-                probs.ctypes.data_as(c_void_p),
-                c_size_t(output_bytes),
-            )
-
-            yield probs.item()
-
-    @staticmethod
-    def from_config(
-        config_path: Union[str, Path],
-        libtensorflowlite_c_path: Union[str, Path],
-    ) -> "OpenWakeWord":
-        config_path = Path(config_path)
-        with open(config_path, "r", encoding="utf-8") as config_file:
-            config = json.load(config_file)
-
-        return OpenWakeWord(
-            id=Path(config["model"]).stem,
-            wake_word=config["wake_word"],
-            tflite_model=config_path.parent / config["model"],
-            libtensorflowlite_c_path=libtensorflowlite_c_path,
-        )
-
-
-# -----------------------------------------------------------------------------
-
-
-class OpenWakeWordFeatures(TfLiteWakeWord):
-    def __init__(
-        self,
-        melspectrogram_model: Union[str, Path],
-        embedding_model: Union[str, Path],
-        libtensorflowlite_c_path: Union[str, Path],
-    ) -> None:
-        TfLiteWakeWord.__init__(self, libtensorflowlite_c_path)
-
-        # Melspectrogram
-        mel_path = str(Path(melspectrogram_model).resolve()).encode("utf-8")
-        self.mel_model = self.lib.TfLiteModelCreateFromFile(mel_path)
-        self.mel_interpreter = self.lib.TfLiteInterpreterCreate(self.mel_model, None)
-
-        mels_dims = (c_int32 * len(MEL_SHAPE))(*MEL_SHAPE)
-        self.lib.TfLiteInterpreterResizeInputTensor(
-            self.mel_interpreter,
-            c_int32(0),
-            mels_dims,
-            c_int32(len(MEL_SHAPE)),
-        )
-        self.lib.TfLiteInterpreterAllocateTensors(self.mel_interpreter)
-        self.mel_input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
-            self.mel_interpreter, c_int32(0)
-        )
-        self.mel_output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
-            self.mel_interpreter, c_int32(0)
-        )
-
-        # Embedding
-        emb_path = str(Path(embedding_model).resolve()).encode("utf-8")
-        self.emb_model = self.lib.TfLiteModelCreateFromFile(emb_path)
-        self.emb_interpreter = self.lib.TfLiteInterpreterCreate(self.emb_model, None)
-        emb_dims = (c_int32 * len(EMB_SHAPE))(*EMB_SHAPE)
-        self.lib.TfLiteInterpreterResizeInputTensor(
-            self.emb_interpreter,
-            c_int32(0),
-            emb_dims,
-            c_int32(len(EMB_SHAPE)),
-        )
-        self.lib.TfLiteInterpreterAllocateTensors(self.emb_interpreter)
-        self.emb_input_tensor = self.lib.TfLiteInterpreterGetInputTensor(
-            self.emb_interpreter, c_int32(0)
-        )
-        self.emb_output_tensor = self.lib.TfLiteInterpreterGetOutputTensor(
-            self.emb_interpreter, c_int32(0)
-        )
-
-        # State
-        self.new_audio_samples: int = AUTOFILL_SECONDS * SAMPLE_RATE
-        self.audio: np.ndarray = np.zeros(shape=(_MAX_SAMPLES,), dtype=np.float32)
-        self.new_mels: int = 0
-        self.mels: np.ndarray = np.zeros(shape=(MAX_MELS, NUM_MELS), dtype=np.float32)
-
-    def process_streaming(self, audio_chunk: bytes) -> Iterable[np.ndarray]:
-        """Generate embeddings from audio."""
-        chunk_array = np.frombuffer(audio_chunk, dtype=np.int16).astype(np.float32)
-
-        # Shift samples left
-        self.audio[: -len(chunk_array)] = self.audio[len(chunk_array) :]
-
-        # Add new samples to end
-        self.audio[-len(chunk_array) :] = chunk_array
-        self.new_audio_samples = min(
-            len(self.audio),
-            self.new_audio_samples + len(chunk_array),
-        )
-
-        while self.new_audio_samples >= MEL_SAMPLES:
-            audio_tensor = np.zeros(shape=(BATCH_SIZE, MEL_SAMPLES), dtype=np.float32)
-            audio_tensor[0, :] = self.audio[
-                -self.new_audio_samples : len(self.audio)
-                - self.new_audio_samples
-                + MEL_SAMPLES
-            ]
-            audio_tensor = np.ascontiguousarray(audio_tensor)
-            self.new_audio_samples = max(0, self.new_audio_samples - SAMPLES_PER_CHUNK)
-
-            # Run inference (mels)
-            audio_ptr = audio_tensor.ctypes.data_as(c_void_p)
-            self.lib.TfLiteTensorCopyFromBuffer(
-                self.mel_input_tensor, audio_ptr, c_size_t(audio_tensor.nbytes)
-            )
-            self.lib.TfLiteInterpreterInvoke(self.mel_interpreter)
-
-            mels_output_bytes = self.lib.TfLiteTensorByteSize(self.mel_output_tensor)
-            mels = np.empty(
-                mels_output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32
-            )
-            self.lib.TfLiteTensorCopyToBuffer(
-                self.mel_output_tensor,
-                mels.ctypes.data_as(c_void_p),
-                c_size_t(mels_output_bytes),
-            )
-
-            mels = (mels / 10) + 2  # transform to fit embedding
-            mels = mels.reshape((1, 1, -1, NUM_MELS))
-
-            # Shift left
-            num_mel_windows = mels.shape[2]
-            self.mels[:-num_mel_windows] = self.mels[num_mel_windows:]
-
-            # Overwrite
-            self.mels[-num_mel_windows:] = mels[0, 0, :, :]
-            self.new_mels = min(len(self.mels), self.new_mels + num_mel_windows)
-
-            while self.new_mels >= EMB_FEATURES:
-                mels_tensor = np.ascontiguousarray(
-                    np.zeros(shape=EMB_SHAPE, dtype=np.float32)
-                )
-                mels_tensor[0, :, :, 0] = self.mels[
-                    -self.new_mels : len(self.mels) - self.new_mels + EMB_FEATURES, :
-                ]
-                self.new_mels = max(0, self.new_mels - EMB_STEP)
-
-                # Run inference (embedding)
-                mels_ptr = mels_tensor.ctypes.data_as(c_void_p)
-                self.lib.TfLiteTensorCopyFromBuffer(
-                    self.emb_input_tensor, mels_ptr, c_size_t(mels_tensor.nbytes)
-                )
-                self.lib.TfLiteInterpreterInvoke(self.emb_interpreter)
-
-                emb_output_bytes = self.lib.TfLiteTensorByteSize(self.emb_output_tensor)
-                emb = np.empty(
-                    emb_output_bytes // np.dtype(np.float32).itemsize, dtype=np.float32
-                )
-                self.lib.TfLiteTensorCopyToBuffer(
-                    self.emb_output_tensor,
-                    emb.ctypes.data_as(c_void_p),
-                    c_size_t(emb_output_bytes),
-                )
-                emb = emb.reshape((1, 1, -1, WW_FEATURES))
-                yield emb

linux_voice_assistant/satellite.py

@@ -30,12 +30,12 @@ from aioesphomeapi.model import (
     VoiceAssistantTimerEventType,
 )
 from google.protobuf import message
+from pymicro_wakeword import MicroWakeWord
+from pyopen_wakeword import OpenWakeWord
 
 from .api_server import APIServer
 from .entity import MediaPlayerEntity
-from .microwakeword import MicroWakeWord
 from .models import ServerState
-from .openwakeword import OpenWakeWord
 from .util import call_all
 
 _LOGGER = logging.getLogger(__name__)
@@ -107,7 +107,7 @@ class VoiceSatelliteProtocol(APIServer):
         _LOGGER.debug("Timer event: type=%s", event_type.name)
         if event_type == VoiceAssistantTimerEventType.VOICE_ASSISTANT_TIMER_FINISHED:
             if not self._timer_finished:
-                self.state.stop_word.is_active = True
+                self.state.active_wake_words.add(self.state.stop_word.id)
                 self._timer_finished = True
                 self.duck()
                 self._play_timer_finished()
@@ -131,7 +131,7 @@ class VoiceSatelliteProtocol(APIServer):
 
             urls.append(msg.media_id)
 
-            self.state.stop_word.is_active = True
+            self.state.active_wake_words.add(self.state.stop_word.id)
             self._continue_conversation = msg.start_conversation
 
             self.duck()
@@ -177,7 +177,9 @@ class VoiceSatelliteProtocol(APIServer):
                     for ww in self.state.available_wake_words.values()
                 ],
                 active_wake_words=[
-                    ww.id for ww in self.state.wake_words.values() if ww.is_active
+                    ww.id
+                    for ww in self.state.wake_words.values()
+                    if ww.id in self.state.active_wake_words
                 ],
                 max_active_wake_words=2,
             )
@@ -196,18 +198,14 @@ class VoiceSatelliteProtocol(APIServer):
                 if not model_info:
                     continue
 
-                _LOGGER.debug("Loading wake word: %s", model_info.config_path)
+                _LOGGER.debug("Loading wake word: %s", model_info.wake_word_path)
-                self.state.wake_words[wake_word_id] = model_info.load(
+                self.state.wake_words[wake_word_id] = model_info.load()
-                    self.state.libtensorflowlite_c_path
-                )
 
                 _LOGGER.info("Wake word set: %s", wake_word_id)
                 active_wake_words.add(wake_word_id)
                 break
 
-            for wake_word in self.state.wake_words.values():
+            self.state.active_wake_words = active_wake_words
-                wake_word.is_active = wake_word.id in active_wake_words
-
             _LOGGER.debug("Active wake words: %s", active_wake_words)
 
             self.state.preferences.active_wake_words = list(active_wake_words)
@@ -239,7 +237,7 @@ class VoiceSatelliteProtocol(APIServer):
         self.state.tts_player.play(self.state.wakeup_sound)
 
     def stop(self) -> None:
-        self.state.stop_word.is_active = False
+        self.state.active_wake_words.discard(self.state.stop_word.id)
         self.state.tts_player.stop()
 
         if self._timer_finished:
@@ -256,7 +254,7 @@ class VoiceSatelliteProtocol(APIServer):
         self._tts_played = True
         _LOGGER.debug("Playing TTS response: %s", self._tts_url)
 
-        self.state.stop_word.is_active = True
+        self.state.active_wake_words.add(self.state.stop_word.id)
         self.state.tts_player.play(self._tts_url, done_callback=self._tts_finished)
 
     def duck(self) -> None:
@@ -268,7 +266,7 @@ class VoiceSatelliteProtocol(APIServer):
         self.state.music_player.unduck()
 
     def _tts_finished(self) -> None:
-        self.state.stop_word.is_active = False
+        self.state.active_wake_words.discard(self.state.stop_word.id)
         self.send_messages([VoiceAssistantAnnounceFinished()])
 
         if self._continue_conversation:

linux_voice_assistant/wakeword.py

@@ -1,89 +0,0 @@
-"""Base class for wake words."""
-
-import ctypes as C
-import os
-from pathlib import Path
-from typing import Union
-
-TfLiteStatus = C.c_int  # kTfLiteOk == 0
-
-
-class TfLiteQuantizationParams(C.Structure):
-    _fields_ = [("scale", C.c_float), ("zero_point", C.c_int32)]
-
-
-c_void_p = C.c_void_p
-c_int32 = C.c_int32
-c_size_t = C.c_size_t
-
-
-def _not_null(result, func, args):
-    if not result:
-        raise RuntimeError(f"{func.__name__} returned NULL")
-    return result
-
-
-class TfLiteWakeWord:
-    def __init__(self, libtensorflowlite_c_path: Union[str, Path]):
-        self.libtensorflowlite_c_path = Path(libtensorflowlite_c_path)
-        # Load ONCE with RTLD_GLOBAL
-        self.lib = C.CDLL(
-            str(self.libtensorflowlite_c_path.resolve()), mode=os.RTLD_GLOBAL
-        )
-
-        lib = self.lib
-
-        # Model / Interpreter
-        lib.TfLiteModelCreateFromFile.argtypes = [C.c_char_p]
-        lib.TfLiteModelCreateFromFile.restype = c_void_p
-
-        lib.TfLiteInterpreterCreate.argtypes = [c_void_p, c_void_p]
-        lib.TfLiteInterpreterCreate.restype = c_void_p
-
-        lib.TfLiteInterpreterAllocateTensors.argtypes = [c_void_p]
-        lib.TfLiteInterpreterAllocateTensors.restype = TfLiteStatus
-
-        lib.TfLiteInterpreterInvoke.argtypes = [c_void_p]
-        lib.TfLiteInterpreterInvoke.restype = TfLiteStatus
-
-        # Tensors: get
-        lib.TfLiteInterpreterGetInputTensor.argtypes = [c_void_p, c_int32]
-        lib.TfLiteInterpreterGetInputTensor.restype = c_void_p
-        lib.TfLiteInterpreterGetInputTensor.errcheck = _not_null
-
-        lib.TfLiteInterpreterGetOutputTensor.argtypes = [c_void_p, c_int32]
-        lib.TfLiteInterpreterGetOutputTensor.restype = c_void_p
-        lib.TfLiteInterpreterGetOutputTensor.errcheck = _not_null
-
-        # Sizes / dims
-        lib.TfLiteTensorByteSize.argtypes = [c_void_p]
-        lib.TfLiteTensorByteSize.restype = c_size_t
-
-        lib.TfLiteTensorNumDims.argtypes = [c_void_p]
-        lib.TfLiteTensorNumDims.restype = c_int32
-
-        lib.TfLiteTensorDim.argtypes = [c_void_p, c_int32]
-        lib.TfLiteTensorDim.restype = c_int32
-
-        # Type & quant params
-        lib.TfLiteTensorType.argtypes = [c_void_p]
-        lib.TfLiteTensorType.restype = c_int32
-
-        lib.TfLiteTensorQuantizationParams.argtypes = [c_void_p]
-        lib.TfLiteTensorQuantizationParams.restype = TfLiteQuantizationParams
-
-        # Resize
-        lib.TfLiteInterpreterResizeInputTensor.argtypes = [
-            c_void_p,
-            c_int32,
-            C.POINTER(c_int32),
-            c_int32,
-        ]
-        lib.TfLiteInterpreterResizeInputTensor.restype = TfLiteStatus
-
-        # Copy buffers
-        lib.TfLiteTensorCopyFromBuffer.argtypes = [c_void_p, c_void_p, c_size_t]
-        lib.TfLiteTensorCopyFromBuffer.restype = TfLiteStatus
-
-        lib.TfLiteTensorCopyToBuffer.argtypes = [c_void_p, c_void_p, c_size_t]
-        lib.TfLiteTensorCopyToBuffer.restype = TfLiteStatus

mypy.ini

@@ -7,7 +7,10 @@ ignore_missing_imports = True
 [mypy-aioesphomeapi.api_pb2.*]
 ignore_missing_imports = True
 
-[mypy-pymicro_features.*]
+[mypy-pymicro_wakeword.*]
+ignore_missing_imports = True
+
+[mypy-pyopen_wakeword.*]
 ignore_missing_imports = True
 
 [mypy-sounddevice.*]

pyproject.toml

@@ -28,7 +28,8 @@ dependencies = [
     "aioesphomeapi==37.2.1",
     "sounddevice<1",
     "numpy>=2,<3",
-    "pymicro-features==1.0.0",
+    "pymicro-wakeword>=2,<3",
+    "pyopen-wakeword>=1,<2",
     "python-mpv>=1,<2",
     "zeroconf",
 ]

tests/test_microwakeword.py

@@ -1,43 +0,0 @@
-"""Tests for microWakeWord."""
-
-import wave
-from pathlib import Path
-
-from linux_voice_assistant.microwakeword import MicroWakeWord, MicroWakeWordFeatures
-from linux_voice_assistant.util import is_arm
-
-_TESTS_DIR = Path(__file__).parent
-_REPO_DIR = _TESTS_DIR.parent
-_MICRO_DIR = _REPO_DIR / "wakewords"
-
-if is_arm():
-    _LIB_DIR = _REPO_DIR / "lib" / "linux_arm64"
-else:
-    _LIB_DIR = _REPO_DIR / "lib" / "linux_amd64"
-
-
-libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so"
-
-
-def test_features() -> None:
-    features = MicroWakeWordFeatures(
-        libtensorflowlite_c_path=libtensorflowlite_c_path,
-    )
-    ww = MicroWakeWord.from_config(
-        config_path=_MICRO_DIR / "okay_nabu.json",
-        libtensorflowlite_c_path=libtensorflowlite_c_path,
-    )
-
-    detected = False
-    with wave.open(str(_TESTS_DIR / "ok_nabu.wav"), "rb") as wav_file:
-        assert wav_file.getframerate() == 16000
-        assert wav_file.getsampwidth() == 2
-        assert wav_file.getnchannels() == 1
-
-        for micro_input in features.process_streaming(
-            wav_file.readframes(wav_file.getnframes())
-        ):
-            if ww.process_streaming(micro_input):
-                detected = True
-
-    assert detected

tests/test_openwakeword.py

@@ -1,47 +0,0 @@
-"""Tests for openWakeWord."""
-
-import wave
-from pathlib import Path
-
-from linux_voice_assistant.openwakeword import OpenWakeWordFeatures, OpenWakeWord
-from linux_voice_assistant.util import is_arm
-
-_TESTS_DIR = Path(__file__).parent
-_REPO_DIR = _TESTS_DIR.parent
-_OWW_DIR = _REPO_DIR / "wakewords" / "openWakeWord"
-
-if is_arm():
-    _LIB_DIR = _REPO_DIR / "lib" / "linux_arm64"
-else:
-    _LIB_DIR = _REPO_DIR / "lib" / "linux_amd64"
-
-
-libtensorflowlite_c_path = _LIB_DIR / "libtensorflowlite_c.so"
-
-
-def test_features() -> None:
-    features = OpenWakeWordFeatures(
-        melspectrogram_model=_OWW_DIR / "melspectrogram.tflite",
-        embedding_model=_OWW_DIR / "embedding_model.tflite",
-        libtensorflowlite_c_path=libtensorflowlite_c_path,
-    )
-    ww = OpenWakeWord(
-        id="ok_nabu",
-        wake_word="okay nabu",
-        tflite_model=_OWW_DIR / "ok_nabu_v0.1.tflite",
-        libtensorflowlite_c_path=libtensorflowlite_c_path,
-    )
-
-    max_prob = 0.0
-    with wave.open(str(_TESTS_DIR / "ok_nabu.wav"), "rb") as wav_file:
-        assert wav_file.getframerate() == 16000
-        assert wav_file.getsampwidth() == 2
-        assert wav_file.getnchannels() == 1
-
-        for embeddings in features.process_streaming(
-            wav_file.readframes(wav_file.getnframes())
-        ):
-            for prob in ww.process_streaming(embeddings):
-                max_prob = max(max_prob, prob)
-
-    assert max_prob > 0.5

tests/test_placeholder.py

@@ -0,0 +1,2 @@
+def test_placeholder():
+    pass