From bb18c963c950ba44b93b94e1ad3635f497917294 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 12:58:38 -0700 Subject: [PATCH 01/10] Update to use OVOSSttFactory directly Mark neon_speech.stt module as deprecated Update dependencies to latest stable versions --- neon_speech/service.py | 4 +--- neon_speech/stt.py | 12 +++++++++++- requirements/requirements.txt | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/neon_speech/service.py b/neon_speech/service.py index 47953e5..bf84cd8 100644 --- a/neon_speech/service.py +++ b/neon_speech/service.py @@ -48,9 +48,7 @@ from ovos_dinkum_listener.service import OVOSDinkumVoiceService from ovos_dinkum_listener.voice_loop.voice_loop import ListeningMode -from neon_speech.stt import STTFactory - -ovos_dinkum_listener.plugins.OVOSSTTFactory = STTFactory +from ovos_plugin_manager.stt import OVOSSTTFactory as STTFactory _SERVICE_READY = Event() diff --git a/neon_speech/stt.py b/neon_speech/stt.py index 7842361..1f4b927 100644 --- a/neon_speech/stt.py +++ b/neon_speech/stt.py @@ -30,15 +30,21 @@ from inspect import signature from threading import Event -from neon_utils import LOG +from ovos_utils.log import LOG, log_deprecation from ovos_plugin_manager.stt import OVOSSTTFactory, get_stt_config from ovos_plugin_manager.templates.stt import StreamingSTT from ovos_config.config import Configuration +log_deprecation("This module is deprecated. Import from `ovos_plugin_manager`", + "5.0.0") + class WrappedSTT(StreamingSTT, ABC): def __new__(cls, base_engine, *args, **kwargs): + log_deprecation("This class is deprecated. Use " + "`ovos_plugin_manager.templates.stt.StreamingSTT", + "5.0.0") results_event = kwargs.get("results_event") or Event() # build STT for k in list(kwargs.keys()): @@ -66,6 +72,10 @@ def stream_stop(self): class STTFactory(OVOSSTTFactory): + log_deprecation("This class is deprecated. Use " + "`ovos_plugin_manager.stt.OVOSSTTFactory", + "5.0.0") + @staticmethod def create(config=None, results_event: Event = None): get_stt_config(config) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 4ad807b..cc6eba1 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,4 @@ -ovos-dinkum-listener~=0.0 +ovos-dinkum-listener~=0.2 ovos-bus-client~=0.0,>=0.0.3 ovos-utils~=0.0,>=0.0.30 ovos-plugin-manager~=0.0,>=0.0.23 @@ -9,4 +9,4 @@ ovos-config~=0.0,>=0.0.7 ovos-vad-plugin-webrtcvad~=0.0.1 ovos-ww-plugin-vosk~=0.1 -ovos-microphone-plugin-alsa~=0.0.0 \ No newline at end of file +ovos-microphone-plugin-alsa~=0.1 \ No newline at end of file From cea0b2cb6a815719e9b428a5bf37f1594ab3c932 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 13:00:29 -0700 Subject: [PATCH 02/10] Update neon-utils dependency for resolution --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index cc6eba1..3d08c8d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,7 +4,7 @@ ovos-utils~=0.0,>=0.0.30 ovos-plugin-manager~=0.0,>=0.0.23 click~=8.0 click-default-group~=1.2 -neon-utils[network,audio]~=1.9 +neon-utils[network,audio]~=1.9,>=1.11.1a3 ovos-config~=0.0,>=0.0.7 ovos-vad-plugin-webrtcvad~=0.0.1 From 3b982aa40b6e066a39422db51e7deeb26fd095f8 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 13:07:14 -0700 Subject: [PATCH 03/10] Remove deprecated deepspeech plugin from test dependencies --- requirements/test_requirements.txt | 3 +-- tests/api_method_tests.py | 6 +++--- tests/unit_tests.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/requirements/test_requirements.txt b/requirements/test_requirements.txt index 94bb8f2..70bb5a3 100644 --- a/requirements/test_requirements.txt +++ b/requirements/test_requirements.txt @@ -1,5 +1,4 @@ -neon-stt-plugin-deepspeech_stream_local~=2.0 -neon-stt-plugin-nemo~=0.0,>=0.0.2 +neon-stt-plugin-nemo~=0.0,>=0.0.5a4 ovos-stt-plugin-vosk~=0.1 ovos-stt-plugin-server~=0.0.3 pytest diff --git a/tests/api_method_tests.py b/tests/api_method_tests.py index 2354b52..4383cf2 100644 --- a/tests/api_method_tests.py +++ b/tests/api_method_tests.py @@ -65,9 +65,9 @@ def setUpClass(cls) -> None: use_neon_speech(init_config_dir)() test_config = dict(Configuration()) - test_config["stt"]["module"] = "deepspeech_stream_local" + test_config["stt"]["module"] = "neon-stt-plugin-nemo" test_config["listener"]["VAD"]["module"] = "dummy" - assert test_config["stt"]["module"] == "deepspeech_stream_local" + assert test_config["stt"]["module"] == "neon-stt-plugin-nemo" ready_event = Event() @@ -77,7 +77,7 @@ def _ready(): cls.speech_service = NeonSpeechClient(speech_config=test_config, daemonic=False, bus=cls.bus, ready_hook=_ready) - assert cls.speech_service.config["stt"]["module"] == "deepspeech_stream_local" + assert cls.speech_service.config["stt"]["module"] == "neon-stt-plugin-nemo" cls.speech_service.start() if not ready_event.wait(120): diff --git a/tests/unit_tests.py b/tests/unit_tests.py index d6b8b32..3845018 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -117,7 +117,7 @@ def test_get_stt_from_file(self): AUDIO_FILE_PATH = os.path.join(os.path.dirname( os.path.realpath(__file__)), "audio_files") TEST_CONFIG = use_neon_speech(Configuration)() - TEST_CONFIG["stt"]["module"] = "deepspeech_stream_local" + TEST_CONFIG["stt"]["module"] = "neon-stt-plugin-nemo" bus = FakeBus() bus.connected_event = Event() bus.connected_event.set() From a7c21d7a48b41ccb2e42a5045c050ee20b0dd99d Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 13:14:34 -0700 Subject: [PATCH 04/10] Mark `use_neon_speech` as deprecated Resolve test failures --- neon_speech/service.py | 3 +-- neon_speech/utils.py | 1 + requirements/docker.txt | 2 +- tests/unit_tests.py | 10 +++++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/neon_speech/service.py b/neon_speech/service.py index bf84cd8..0644813 100644 --- a/neon_speech/service.py +++ b/neon_speech/service.py @@ -129,8 +129,7 @@ def __init__(self, ready_hook=on_ready, error_hook=on_error, self.lock = Lock() self._stop_service = Event() if self.config.get('listener', {}).get('enable_stt_api', True): - self.api_stt = STTFactory.create(config=self.config, - results_event=None) + self.api_stt = STTFactory.create(config=self.config) else: LOG.info("Skipping api_stt init") self.api_stt = None diff --git a/neon_speech/utils.py b/neon_speech/utils.py index 6821a42..7369308 100644 --- a/neon_speech/utils.py +++ b/neon_speech/utils.py @@ -104,6 +104,7 @@ def init_stt_plugin(plugin: str): LOG.warning(f"Could not find plugin: {plugin}") +@deprecated("Platform detection has been deprecated", "5.0.0") def use_neon_speech(func): """ Wrapper to ensure call originates from neon_speech for stack checks. diff --git a/requirements/docker.txt b/requirements/docker.txt index bb0aae8..6831c38 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -1,5 +1,5 @@ ovos-stt-plugin-vosk~=0.1 -neon-stt-plugin-nemo~=0.0.2 +neon-stt-plugin-nemo~=0.0.2,>=0.0.5a4 onnxruntime!=1.16.0 # TODO: Patching https://github.com/microsoft/onnxruntime/issues/17631 # Load alternative WW plugins so they are available diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 3845018..b2ed9e3 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -34,7 +34,8 @@ from os.path import dirname, join from threading import Thread, Event -from unittest.mock import Mock, patch +from unittest import skip +from unittest.mock import patch from click.testing import CliRunner from ovos_bus_client import Message @@ -44,6 +45,8 @@ CONFIG_PATH = os.path.join(dirname(__file__), "config") os.environ["XDG_CONFIG_HOME"] = CONFIG_PATH +os.environ["OVOS_CONFIG_BASE_FOLDER"] = "neon" +os.environ["OVOS_CONFIG_FILENAME"] = "neon.yaml" sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) @@ -80,12 +83,13 @@ def test_install_stt_plugin(self): "ovos-stt-plugin-vosk")) import ovos_stt_plugin_vosk + @skip("Configuration patching is deprecated") def test_patch_config(self): from neon_speech.utils import use_neon_speech from neon_utils.configuration_utils import init_config_dir test_config_dir = os.path.join(os.path.dirname(__file__), "config") os.makedirs(test_config_dir, exist_ok=True) - os.environ["XDG_CONFIG_HOME"] = test_config_dir + use_neon_speech(init_config_dir)() with open(join(test_config_dir, "OpenVoiceOS", 'ovos.conf')) as f: @@ -156,7 +160,7 @@ def test_ovos_plugin_compat(self): ovos_vosk_streaming = STTFactory().create( {'module': 'ovos-stt-plugin-vosk-streaming', 'lang': 'en-us'}) - self.assertIsInstance(ovos_vosk_streaming.results_event, Event) + # self.assertIsInstance(ovos_vosk_streaming.results_event, Event) test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "audio_files", "stop.wav") from neon_utils.file_utils import get_audio_file_stream From 5b195ac0a49bb509de8603bdbf96fe84ecfde08d Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 13:26:59 -0700 Subject: [PATCH 05/10] Troubleshooting unit test failures --- tests/unit_tests.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index b2ed9e3..ffae32c 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -230,8 +230,8 @@ def on_ready(cls): @classmethod def setUpClass(cls): from ovos_config.config import update_mycroft_config - from neon_utils.configuration_utils import init_config_dir - init_config_dir() + # from neon_utils.configuration_utils import init_config_dir + # init_config_dir() update_mycroft_config({"hotwords": cls.hotwords_config, "stt": {"module": "neon-stt-plugin-nemo"}, @@ -239,11 +239,9 @@ def setUpClass(cls): import importlib import ovos_config.config importlib.reload(ovos_config.config) - # from ovos_config.config import Configuration - # assert Configuration.xdg_configs[0]['hotwords'] == hotwords_config + from ovos_config.config import Configuration + assert Configuration.xdg_configs[0]['hotwords'] == cls.hotwords_config - from neon_speech.utils import use_neon_speech - use_neon_speech(init_config_dir)() from neon_speech.service import NeonSpeechClient cls.service = NeonSpeechClient(bus=cls.bus, ready_hook=cls.on_ready) # assert Configuration() == service.loop.config_core From a48d1c7248af1bc41919b274a534d515a6d5713f Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 13:55:38 -0700 Subject: [PATCH 06/10] Troubleshooting unit test failures --- tests/unit_tests.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index ffae32c..001eee1 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -31,6 +31,7 @@ import shutil import sys import unittest +import yaml from os.path import dirname, join from threading import Thread, Event @@ -48,7 +49,6 @@ os.environ["OVOS_CONFIG_BASE_FOLDER"] = "neon" os.environ["OVOS_CONFIG_FILENAME"] = "neon.yaml" - sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) @@ -229,13 +229,13 @@ def on_ready(cls): @classmethod def setUpClass(cls): - from ovos_config.config import update_mycroft_config - # from neon_utils.configuration_utils import init_config_dir - # init_config_dir() + os.makedirs(join(CONFIG_PATH, "neon"), exist_ok=True) + test_config = join(CONFIG_PATH, "neon", "neon.yaml") + with open(test_config, 'w+') as f: + yaml.dump({"hotwords": cls.hotwords_config, + "stt": {"module": "neon-stt-plugin-nemo"}, + "VAD": {"module": "dummy"}}, f) - update_mycroft_config({"hotwords": cls.hotwords_config, - "stt": {"module": "neon-stt-plugin-nemo"}, - "VAD": {"module": "dummy"}}) import importlib import ovos_config.config importlib.reload(ovos_config.config) @@ -244,7 +244,8 @@ def setUpClass(cls): from neon_speech.service import NeonSpeechClient cls.service = NeonSpeechClient(bus=cls.bus, ready_hook=cls.on_ready) - # assert Configuration() == service.loop.config_core + + assert cls.service.reload_configuration in Configuration._callbacks def _mocked_run(): stopping_event = Event() From 523e22d580aa63221569c1ae16a199c285f19f88 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 14:10:27 -0700 Subject: [PATCH 07/10] Patch configuration reload --- neon_speech/service.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/neon_speech/service.py b/neon_speech/service.py index 0644813..7b9c2aa 100644 --- a/neon_speech/service.py +++ b/neon_speech/service.py @@ -220,6 +220,16 @@ def register_event_handlers(self): self.bus.on("neon.enable_wake_word", self.handle_enable_wake_word) self.bus.on("neon.disable_wake_word", self.handle_disable_wake_word) + # TODO: Patching config reload behavior + self.bus.on("configuration.patch", self._patch_handle_config_reload) + + def _patch_handle_config_reload(self, _: Message): + # This patches observed behavior where the filewatcher fails to trigger. + # Configuration reload is idempotent, so calling it again will have + # minimal impact + self.config.reload() + self.reload_configuration() + def _handle_get_languages_stt(self, message): if self.config.get('listener', {}).get('enable_voice_loop', True): return OVOSDinkumVoiceService._handle_get_languages_stt(self, From d61b32ca7266d467c8fa85f0f61646bef9f0b078 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 15:45:21 -0700 Subject: [PATCH 08/10] Update API methods to use `transcribe` method with confidence levels --- neon_speech/service.py | 22 +++++++++++++--------- tests/unit_tests.py | 6 ++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/neon_speech/service.py b/neon_speech/service.py index 7b9c2aa..3144fdb 100644 --- a/neon_speech/service.py +++ b/neon_speech/service.py @@ -27,7 +27,7 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -from typing import Dict +from typing import Dict, List, Tuple import ovos_dinkum_listener.plugins @@ -421,9 +421,11 @@ def handle_get_stt(self, message: Message): message.context['timing']['client_to_core'] = \ received_time - sent_time message.context['timing']['response_sent'] = time() + transcribed_str = [t[0] for t in transcriptions] self.bus.emit(message.reply(ident, data={"parser_data": parser_data, - "transcripts": transcriptions})) + "transcripts": transcribed_str, + "transcripts_with_conf": transcriptions})) except Exception as e: LOG.error(e) message.context['timing']['response_sent'] = time() @@ -474,8 +476,9 @@ def build_context(msg: Message): message.context.setdefault('timing', dict()) message.context['timing'] = {**timing, **message.context['timing']} context = build_context(message) + transribed_str = [t[0] for t in transcriptions] data = { - "utterances": transcriptions, + "utterances": transribed_str, "lang": message.data.get("lang", "en-us") } # Send a new message to the skills module with proper routing ctx @@ -485,7 +488,8 @@ def build_context(msg: Message): # Reply to original message with transcription/audio parser data self.bus.emit(message.reply(ident, data={"parser_data": parser_data, - "transcripts": transcriptions, + "transcripts": transribed_str, + "transcripts_with_conf": transcriptions, "skills_recv": handled})) except Exception as e: LOG.error(e) @@ -535,7 +539,7 @@ def _write_encoded_file(audio_data: str) -> str: return wav_file_path def _get_stt_from_file(self, wav_file: str, - lang: str = None) -> (AudioData, dict, list): + lang: str = None) -> (AudioData, dict, List[Tuple[str, float]]): """ Performs STT and audio processing on the specified wav_file :param wav_file: wav audio file to process @@ -569,18 +573,18 @@ def _get_stt_from_file(self, wav_file: str, self.api_stt.stream_data(data) except EOFError: break - transcriptions = self.api_stt.stream_stop() + transcriptions = self.api_stt.transcribe(None, None) self.lock.release() else: LOG.error(f"Timed out acquiring lock, not processing: {wav_file}") transcriptions = [] else: - transcriptions = self.api_stt.execute(audio_data, lang) + transcriptions = self.api_stt.transcribe(audio_data, lang) if isinstance(transcriptions, str): - LOG.warning("Transcriptions is a str, no alternatives provided") + LOG.error("Transcriptions is a str, no alternatives provided") transcriptions = [transcriptions] - transcriptions = [clean_quotes(t) for t in transcriptions] + transcriptions = [(clean_quotes(t[0]), t[1]) for t in transcriptions] get_stt = float(_stopwatch.time) with _stopwatch: diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 001eee1..fc02d2d 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -133,7 +133,8 @@ def test_get_stt_from_file(self): self.assertIsInstance(audio, AudioData) self.assertIsInstance(context, dict) self.assertIsInstance(transcripts, list) - self.assertIn("stop", transcripts) + tr_str = [t[0] for t in transcripts] + self.assertIn("stop", tr_str) def threaded_get_stt(): audio, context, transcripts = \ @@ -141,7 +142,8 @@ def threaded_get_stt(): self.assertIsInstance(audio, AudioData) self.assertIsInstance(context, dict) self.assertIsInstance(transcripts, list) - self.assertIn("stop", transcripts) + tr_str = [t[0] for t in transcripts] + self.assertIn("stop", tr_str) threads = list() for i in range(0, 12): From 72cbd60f2b3eaa350528a3319f4f7b080bba9810 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 16:19:08 -0700 Subject: [PATCH 09/10] Update minimum OPM to support `transcribe` method for STT plugins Update nemo plugin to include `transcribe` method --- requirements/docker.txt | 2 +- requirements/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/docker.txt b/requirements/docker.txt index 6831c38..4db7f2a 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -1,5 +1,5 @@ ovos-stt-plugin-vosk~=0.1 -neon-stt-plugin-nemo~=0.0.2,>=0.0.5a4 +neon-stt-plugin-nemo~=0.0.2,>=0.0.5a5 onnxruntime!=1.16.0 # TODO: Patching https://github.com/microsoft/onnxruntime/issues/17631 # Load alternative WW plugins so they are available diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 3d08c8d..075f340 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,7 +1,7 @@ ovos-dinkum-listener~=0.2 ovos-bus-client~=0.0,>=0.0.3 ovos-utils~=0.0,>=0.0.30 -ovos-plugin-manager~=0.0,>=0.0.23 +ovos-plugin-manager~=0.1 click~=8.0 click-default-group~=1.2 neon-utils[network,audio]~=1.9,>=1.11.1a3 From e7928473a7ba770f2cc19d0bcf1973911fd08733 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 19 Sep 2024 16:30:52 -0700 Subject: [PATCH 10/10] Update OPM dependency to allow installation with current neon-core --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 075f340..cf7822e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,7 +1,7 @@ ovos-dinkum-listener~=0.2 ovos-bus-client~=0.0,>=0.0.3 ovos-utils~=0.0,>=0.0.30 -ovos-plugin-manager~=0.1 +ovos-plugin-manager~=0.0,>=0.0.26a39 click~=8.0 click-default-group~=1.2 neon-utils[network,audio]~=1.9,>=1.11.1a3