From d954512d0a6d62b28d81e32f82d8196f054cbb7f Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Thu, 24 Aug 2023 17:16:38 +0300 Subject: [PATCH 1/2] mycroft-precise --- README.md | 1 + .../mycroft_precise/bin/precise_server.py | 140 ++++++++++++++++++ .../mycroft_precise/bin/precise_stream.py | 80 ++++++++++ .../wake/mycroft_precise/requirements.txt | 1 + .../wake/mycroft_precise/script/download.py | 61 ++++++++ programs/wake/mycroft_precise/script/server | 20 +++ programs/wake/mycroft_precise/script/setup | 47 ++++++ rhasspy3/configuration.yaml | 19 +++ 8 files changed, 369 insertions(+) create mode 100755 programs/wake/mycroft_precise/bin/precise_server.py create mode 100755 programs/wake/mycroft_precise/bin/precise_stream.py create mode 100644 programs/wake/mycroft_precise/requirements.txt create mode 100755 programs/wake/mycroft_precise/script/download.py create mode 100755 programs/wake/mycroft_precise/script/server create mode 100755 programs/wake/mycroft_precise/script/setup diff --git a/README.md b/README.md index 14c8c4f..1b28f0f 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ See `servers` section of `configuration.yaml` file. * wake * [porcupine1](https://github.com/Picovoice/porcupine) * [precise-lite](https://github.com/mycroftAI/mycroft-precise) + * [mycroft-precise](https://github.com/mycroftAI/mycroft-precise) * [snowboy](https://github.com/Kitt-AI/snowboy) * vad * [silero](https://github.com/snakers4/silero-vad) diff --git a/programs/wake/mycroft_precise/bin/precise_server.py b/programs/wake/mycroft_precise/bin/precise_server.py new file mode 100755 index 0000000..c44f301 --- /dev/null +++ b/programs/wake/mycroft_precise/bin/precise_server.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +import argparse +import logging +import os +import socket +from pathlib import Path + +from precise_runner import PreciseEngine, PreciseRunner, ReadWriteStream + +from rhasspy3.audio import AudioChunk, AudioStart, AudioStop +from rhasspy3.event import read_event, write_event +from rhasspy3.wake import Detection, NotDetected + +_FILE = Path(__file__) +_DIR = _FILE.parent +_LOGGER = logging.getLogger(_FILE.stem) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("model", help="Location to .pb model file to use (with .pb.params)") + parser.add_argument( + "--socketfile", required=True, help="Path to Unix domain socket file" + ) + parser.add_argument("--engine", default='.venv/bin/precise-engine', help="Path to precise-engine executable") + parser.add_argument("--chunk_size", type=int, default=2048, help="Number of *bytes* per prediction. Higher numbers decrease CPU usage but increase latency") + parser.add_argument("--trigger_level", type=int, default=3, help="Number of chunk activations needed to trigger detection event. Higher values add latency but reduce false positives") + parser.add_argument("--sensitivity", type=float, default=0.5, help="From 0.0 to 1.0, how sensitive the network should be") + parser.add_argument("--debug", action="store_true", help="Log DEBUG messages") + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) + + # Need to unlink socket if it exists + try: + os.unlink(args.socketfile) + except OSError: + pass + + try: + # Create socket server + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(args.socketfile) + sock.listen() + + # Load converted faster-whisper model + engine = PreciseEngine(args.engine, args.model, args.chunk_size) + stream = ReadWriteStream() + + conn_file_s = None + is_detected = False + timestamp = None + name = os.path.basename(args.model).removesuffix(".pb") + sensitivity = args.sensitivity + + def activation(): + nonlocal is_detected, timestamp, conn_file_s, name + + _LOGGER.info("Detected") + + if is_detected: + return + if conn_file_s is None: + return + + write_event( + Detection( + name=name, timestamp=timestamp + ).event(), + conn_file_s, + ) # type: ignore + + is_detected = True + + def prediction(prob): + nonlocal is_detected, conn_file_s + + if not is_detected: + return + if conn_file_s is None: + return + if prob >= sensitivity: + return + + write_event(NotDetected().event(), conn_file_s) # type: ignore + + is_detected = False + + runner = PreciseRunner(engine, trigger_level=args.trigger_level, sensitivity=args.sensitivity, stream=stream, on_activation=activation, on_prediction=prediction) + runner.start() + _LOGGER.info("Ready") + + # Listen for connections + while True: + try: + connection, client_address = sock.accept() + _LOGGER.debug("Connection from %s", client_address) + + with connection, connection.makefile(mode="rwb") as conn_file: + conn_file_s = conn_file + while True: + event = read_event(conn_file) # type: ignore + if event is None: + break + + if AudioStart.is_type(event.type): + _LOGGER.debug("Receiving audio") + continue + + if AudioStop.is_type(event.type): + _LOGGER.debug("Audio stopped") + break + + if not AudioChunk.is_type(event.type): + continue + + chunk = AudioChunk.from_event(event) + + timestamp = chunk.timestamp + stream.write(chunk.audio) + + if is_detected: + write_event(NotDetected().event(), conn_file) # type: ignore + + except KeyboardInterrupt: + break + except Exception: + _LOGGER.exception("Error communicating with socket client") + finally: + conn_file_s = None + is_detected = False + finally: + os.unlink(args.socketfile) + runner.stop() + + +# ----------------------------------------------------------------------------- + +if __name__ == "__main__": + main() diff --git a/programs/wake/mycroft_precise/bin/precise_stream.py b/programs/wake/mycroft_precise/bin/precise_stream.py new file mode 100755 index 0000000..c4af393 --- /dev/null +++ b/programs/wake/mycroft_precise/bin/precise_stream.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import argparse +import logging +import os +from pathlib import Path + +from precise_runner import PreciseEngine, PreciseRunner, ReadWriteStream + +from rhasspy3.audio import AudioChunk, AudioStop +from rhasspy3.event import read_event, write_event +from rhasspy3.wake import Detection, NotDetected + +_FILE = Path(__file__) +_DIR = _FILE.parent +_LOGGER = logging.getLogger(_FILE.stem) + +# ----------------------------------------------------------------------------- + + +def main() -> None: + """Main method.""" + parser = argparse.ArgumentParser() + parser.add_argument("model", help="Location to .pb model file to use (with .pb.params)") + parser.add_argument("--engine", default='.venv/bin/precise-engine', help="Path to precise-engine executable") + parser.add_argument("--chunk_size", type=int, default=2048, help="Number of *bytes* per prediction. Higher numbers decrease CPU usage but increase latency") + parser.add_argument("--trigger_level", type=int, default=3, help="Number of chunk activations needed to trigger detection event. Higher values add latency but reduce false positives") + parser.add_argument("--sensitivity", type=float, default=0.5, help="From 0.0 to 1.0, how sensitive the network should be") + parser.add_argument("--debug", action="store_true", help="Log DEBUG messages") + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) + + engine = PreciseEngine(args.engine, args.model, args.chunk_size) + stream = ReadWriteStream() + + is_detected = False + timestamp = None + name = os.path.basename(args.model).removesuffix(".pb") + + def activation(): + nonlocal is_detected, timestamp, name + write_event( + Detection( + name=name, timestamp=timestamp + ).event() + ) + is_detected = True + + + runner = PreciseRunner(engine, trigger_level=args.trigger_level, sensitivity=args.sensitivity, stream=stream, on_activation=activation) + runner.start() + + try: + while True: + event = read_event() + if event is None: + break + + if AudioStop.is_type(event.type): + break + + if not AudioChunk.is_type(event.type): + continue + + chunk = AudioChunk.from_event(event) + + timestamp = chunk.timestamp + stream.write(chunk.audio) + + if is_detected: + write_event(NotDetected().event()) + except KeyboardInterrupt: + pass + + runner.stop() + +# ----------------------------------------------------------------------------- + +if __name__ == "__main__": + main() diff --git a/programs/wake/mycroft_precise/requirements.txt b/programs/wake/mycroft_precise/requirements.txt new file mode 100644 index 0000000..57b038f --- /dev/null +++ b/programs/wake/mycroft_precise/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/MycroftAI/mycroft-precise.git@refs/pull/242/merge diff --git a/programs/wake/mycroft_precise/script/download.py b/programs/wake/mycroft_precise/script/download.py new file mode 100755 index 0000000..a53459d --- /dev/null +++ b/programs/wake/mycroft_precise/script/download.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import argparse +import logging +import re +import subprocess +import tarfile +from pathlib import Path +from urllib.request import urlopen + +_DIR = Path(__file__).parent +_LOGGER = logging.getLogger("download") + + +def get_models(): + _LOGGER.info("Downloading list of models...") + url = "https://github.com/MycroftAI/Precise-Community-Data.git/branches/master" + url2 = "https://github.com/MycroftAI/Precise-Community-Data/raw/master/" + filelist = subprocess.run(["svn", "ls", "-R", url], capture_output=True) + result = {} + for file in [x.decode() for x in filelist.stdout.split(b'\n')]: + if not file.endswith('.tar.gz') or not "/models/" in file: + continue + result.update({re.split('[0-9]', file)[0].split('/')[-1].removesuffix('-'): url2 + file}) + return result + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--destination", help="Path to destination directory (default: data)" + ) + parser.add_argument( + "--name", + help="Model name", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + + if args.destination: + args.destination = Path(args.destination) + else: + # Assume we're in programs/wake/mycroft_precise/script + data_dir = _DIR.parent.parent.parent.parent / "data" + args.destination = data_dir / "wake" / "mycroft_precise" + + args.destination.parent.mkdir(parents=True, exist_ok=True) + + models = get_models() + if args.name is None or args.name not in models: + _LOGGER.info("Available models: %s", list(models.keys())) + return + + _LOGGER.info("Downloading %s", args.name) + _LOGGER.info("URL: %s", models[args.name]) + with urlopen(models[args.name]) as response: + with tarfile.open(mode="r|*", fileobj=response) as tar_gz: + _LOGGER.info("Extracting to %s", args.destination) + tar_gz.extractall(args.destination) + + +if __name__ == "__main__": + main() diff --git a/programs/wake/mycroft_precise/script/server b/programs/wake/mycroft_precise/script/server new file mode 100755 index 0000000..9fe2f18 --- /dev/null +++ b/programs/wake/mycroft_precise/script/server @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Directory of *this* script +this_dir="$( cd "$( dirname "$0" )" && pwd )" + +# Base directory of repo +base_dir="$(realpath "${this_dir}/..")" + +# Path to virtual environment +: "${venv:=${base_dir}/.venv}" + +if [ -d "${venv}" ]; then + source "${venv}/bin/activate" +fi + +socket_dir="${base_dir}/var/run" +mkdir -p "${socket_dir}" + +python3 "${base_dir}/bin/precise_server.py" --socketfile "${socket_dir}/precise.socket" "$@" diff --git a/programs/wake/mycroft_precise/script/setup b/programs/wake/mycroft_precise/script/setup new file mode 100755 index 0000000..23572ec --- /dev/null +++ b/programs/wake/mycroft_precise/script/setup @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Directory of *this* script +this_dir="$( cd "$( dirname "$0" )" && pwd )" + +# Base directory of repo +base_dir="$(realpath "${this_dir}/..")" + +# Path to virtual environment +: "${venv:=${base_dir}/.venv}" + +# Python binary to use +: "${PYTHON=python3}" + +python_version="$(${PYTHON} --version)" + +if [ ! -d "${venv}" ]; then + # Create virtual environment + echo "Creating virtual environment at ${venv} (${python_version})" + rm -rf "${venv}" + "${PYTHON}" -m venv "${venv}" + source "${venv}/bin/activate" + + pip3 install --upgrade pip + pip3 install --upgrade wheel setuptools +else + source "${venv}/bin/activate" +fi + +# Install Python dependencies +echo 'Installing Python dependencies' +pip3 install -r "${base_dir}/requirements.txt" + +# Download official models +URL="https://github.com/MycroftAI/precise-data.git/branches/models" +DEST="$this_dir/../../../../data/wake/mycroft_precise" +echo "Downloading models" +mkdir -p "$DEST" +for MODEL in $(svn ls "$URL" | grep '\.pb'); do + echo "$MODEL" + svn cat "$URL/$MODEL" > "$DEST/$MODEL" +done + +# ----------------------------------------------------------------------------- + +echo "OK" diff --git a/rhasspy3/configuration.yaml b/rhasspy3/configuration.yaml index 73b068b..94c9cb3 100644 --- a/rhasspy3/configuration.yaml +++ b/rhasspy3/configuration.yaml @@ -78,6 +78,19 @@ programs: template_args: model: "share/hey_mycroft.tflite" + # https://github.com/mycroftAI/mycroft-precise + # Model included in data/ + # Download community models with script/download.py + mycroft_precise: + command: | + .venv/bin/python3 bin/precise_stream.py "${model}" + template_args: + model: "${data_dir}/hey-mycroft.pb" + + mycroft_precise.client: + command: | + client_unix_socket.py var/run/precise.socket + # TODO: snowman # https://github.com/Thalhammer/snowman/ @@ -450,6 +463,12 @@ programs: # ----------------------------------------------------------------------------- servers: + wake: + mycroft_precise: + command: | + script/server "${model}" + template_args: + model: "${data_dir}/hey-mycroft.pb" asr: vosk: command: | From 17c3a90b367379c34d12cd26cf44808e436a1090 Mon Sep 17 00:00:00 2001 From: Denis Shulyaka Date: Thu, 24 Aug 2023 20:56:11 +0300 Subject: [PATCH 2/2] wrap callback into try-except --- .../mycroft_precise/bin/precise_server.py | 24 ++++++++++++------- .../mycroft_precise/bin/precise_stream.py | 15 +++++++----- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/programs/wake/mycroft_precise/bin/precise_server.py b/programs/wake/mycroft_precise/bin/precise_server.py index c44f301..1e0f56c 100755 --- a/programs/wake/mycroft_precise/bin/precise_server.py +++ b/programs/wake/mycroft_precise/bin/precise_server.py @@ -63,14 +63,17 @@ def activation(): if conn_file_s is None: return - write_event( - Detection( - name=name, timestamp=timestamp - ).event(), - conn_file_s, - ) # type: ignore - - is_detected = True + try: + write_event( + Detection( + name=name, timestamp=timestamp + ).event(), + conn_file_s, + ) # type: ignore + + is_detected = True + except Exception: + pass def prediction(prob): nonlocal is_detected, conn_file_s @@ -82,7 +85,10 @@ def prediction(prob): if prob >= sensitivity: return - write_event(NotDetected().event(), conn_file_s) # type: ignore + try: + write_event(NotDetected().event(), conn_file_s) # type: ignore + except Exception: + pass is_detected = False diff --git a/programs/wake/mycroft_precise/bin/precise_stream.py b/programs/wake/mycroft_precise/bin/precise_stream.py index c4af393..b8b15a3 100755 --- a/programs/wake/mycroft_precise/bin/precise_stream.py +++ b/programs/wake/mycroft_precise/bin/precise_stream.py @@ -39,12 +39,15 @@ def main() -> None: def activation(): nonlocal is_detected, timestamp, name - write_event( - Detection( - name=name, timestamp=timestamp - ).event() - ) - is_detected = True + try: + write_event( + Detection( + name=name, timestamp=timestamp + ).event() + ) + is_detected = True + except Exception: + pass runner = PreciseRunner(engine, trigger_level=args.trigger_level, sensitivity=args.sensitivity, stream=stream, on_activation=activation)