diff --git a/jasper/application.py b/jasper/application.py index 080c7e03b..79538b12a 100644 --- a/jasper/application.py +++ b/jasper/application.py @@ -105,10 +105,18 @@ def __init__(self, use_mic=USE_STANDARD_MIC, batch_file=None): tts_slug = self.config['tts_engine'] except KeyError: tts_slug = 'espeak-tts' - self._logger.warning("tts_engine not specified in profile, using" + + self._logger.warning("tts_engine not specified in profile, using " + "defaults.") self._logger.debug("Using TTS engine '%s'", tts_slug) + try: + tti_slug = self.config['tti_engine'] + except KeyError: + tti_slug = 'phrasematcher-tti' + self._logger.warning("tti_engine not specified in profile, using " + + "defaults.") + self._logger.debug("Using TTI engine '%s'", tti_slug) + try: keyword = self.config['keyword'] except KeyError: @@ -172,33 +180,11 @@ def __init__(self, use_mic=USE_STANDARD_MIC, batch_file=None): ', '.join(devices)) raise - # Initialize Brain - self.brain = brain.Brain(self.config) - for info in self.plugins.get_plugins_by_category('speechhandler'): - try: - plugin = info.plugin_class(info, self.config) - except Exception as e: - self._logger.warning( - "Plugin '%s' skipped! (Reason: %s)", info.name, - e.message if hasattr(e, 'message') else 'Unknown', - exc_info=( - self._logger.getEffectiveLevel() == logging.DEBUG)) - else: - self.brain.add_plugin(plugin) - - if len(self.brain.get_plugins()) == 0: - msg = 'No plugins for handling speech found!' - self._logger.error(msg) - raise RuntimeError(msg) - elif len(self.brain.get_all_phrases()) == 0: - msg = 'No command phrases found!' - self._logger.error(msg) - raise RuntimeError(msg) - + # create instanz of SST and TTS active_stt_plugin_info = self.plugins.get_plugin( active_stt_slug, category='stt') active_stt_plugin = active_stt_plugin_info.plugin_class( - 'default', self.brain.get_plugin_phrases(), active_stt_plugin_info, + active_stt_plugin_info, self.config) if passive_stt_slug != active_stt_slug: @@ -208,7 +194,6 @@ def __init__(self, use_mic=USE_STANDARD_MIC, batch_file=None): passive_stt_plugin_info = active_stt_plugin_info passive_stt_plugin = passive_stt_plugin_info.plugin_class( - 'keyword', self.brain.get_standard_phrases() + [keyword], passive_stt_plugin_info, self.config) tts_plugin_info = self.plugins.get_plugin(tts_slug, category='tts') @@ -229,6 +214,41 @@ def __init__(self, use_mic=USE_STANDARD_MIC, batch_file=None): passive_stt_plugin, active_stt_plugin, tts_plugin, self.config, keyword=keyword) + # Text-to-intent handler + tti_plugin_info = self.plugins.get_plugin(tti_slug, category='tti') + #tti_plugin = tti_plugin_info.plugin_class(tti_plugin_info, self.config) + + # Initialize Brain + self.brain = brain.Brain(self.config, + tti_plugin_info.plugin_class(tti_plugin_info, self.config)) + for info in self.plugins.get_plugins_by_category('speechhandler'): + # create instanz + try: + plugin = info.plugin_class(info, self.config, + tti_plugin_info.plugin_class(tti_plugin_info, self.config), self.mic) + except Exception as e: + self._logger.warning( + "Plugin '%s' skipped! (Reason: %s)", info.name, + e.message if hasattr(e, 'message') else 'Unknown', + exc_info=( + self._logger.getEffectiveLevel() == logging.DEBUG)) + else: + self.brain.add_plugin(plugin) + + if len(self.brain.get_plugins()) == 0: + msg = 'No plugins for handling speech found!' + self._logger.error(msg) + raise RuntimeError(msg) + elif len(self.brain.get_all_phrases()) == 0: + msg = 'No command phrases found!' + self._logger.error(msg) + raise RuntimeError(msg) + + # init SSTs and compile vocabulary if needed + active_stt_plugin.init('default', self.brain.get_plugin_phrases()) + passive_stt_plugin.init('keyword', self.brain.get_standard_phrases() + [keyword]) + + # Initialize Conversation self.conversation = conversation.Conversation( self.mic, self.brain, self.config) diff --git a/jasper/brain.py b/jasper/brain.py index e227a66ea..6aabc5d76 100644 --- a/jasper/brain.py +++ b/jasper/brain.py @@ -4,17 +4,19 @@ class Brain(object): - def __init__(self, config): + def __init__(self, config, tti_plugin): """ Instantiates a new Brain object, which cross-references user - input with a list of modules. Note that the order of brain.modules - matters, as the Brain will return the first module + input with a list of modules. Note that the order of brain.plugins + matters, as the Brain will return the first plugin that accepts a given input. """ self._plugins = [] self._logger = logging.getLogger(__name__) self._config = config + self._tti_plugin = tti_plugin + #self._tti = tti_plugin_info.plugin_class(tti_plugin_info, self._config) def add_plugin(self, plugin): self._plugins.append(plugin) diff --git a/jasper/plugin.py b/jasper/plugin.py index 4db7f5bbb..5545f0371 100644 --- a/jasper/plugin.py +++ b/jasper/plugin.py @@ -29,37 +29,99 @@ class AudioEnginePlugin(GenericPlugin, audioengine.AudioEngine): class SpeechHandlerPlugin(GenericPlugin, i18n.GettextMixin): + """ + Generic parent class for SpeechHandlingPlugins + """ __metaclass__ = abc.ABCMeta - def __init__(self, *args, **kwargs): - GenericPlugin.__init__(self, *args, **kwargs) + def __init__(self, info, config, tti_plugin, mic): + """ + Instantiates a new generic SpeechhandlerPlugin instance. Requires a tti_plugin and a mic + instance. + """ + GenericPlugin.__init__(self, info, config) i18n.GettextMixin.__init__( self, self.info.translations, self.profile) + self._tti_plugin = tti_plugin + #self._tti_plugin = tti_plugin_info.plugin_class(tti_plugin_info, self._plugin_config) + self._mic = mic + +# @classmethod +# def init(self, *args, **kwargs): +# """ +# Initiate Plugin, e.g. do some runtime preparation stuff +# +# Arguments: +# """ +# self._tti_plugin.init(self, *args, **kwargs) - @abc.abstractmethod + @classmethod def get_phrases(self): - pass + return self._tti_plugin.get_phrases(self) + @classmethod @abc.abstractmethod def handle(self, text, mic): pass - @abc.abstractmethod + @classmethod def is_valid(self, text): - pass + return self._tti_plugin.is_valid(self, text) + + @classmethod + def check_phrase(self, text): + return self._tti_plugin.get_confidence(self, text) def get_priority(self): return 0 +class TTIPlugin(GenericPlugin): + """ + Generic parent class for text-to-intent handler + """ + __metaclass__ = abc.ABCMeta + ACTIONS = [] + WORDS = {} + + def __init__(self, *args, **kwargs): + GenericPlugin.__init__(self, *args, **kwargs) + + @classmethod + @abc.abstractmethod + def get_phrases(cls): + pass + + @classmethod + @abc.abstractmethod + def get_intent(cls, phrase): + pass + + @abc.abstractmethod + def is_valid(self, phrase): + pass + + @classmethod + def get_confidence(self, phrase): + return self.is_valid(self, phrase) + + @abc.abstractmethod + def get_actionlist(self, phrase): + pass + + class STTPlugin(GenericPlugin): - def __init__(self, name, phrases, *args, **kwargs): + def __init__(self, *args, **kwargs): GenericPlugin.__init__(self, *args, **kwargs) - self._vocabulary_phrases = phrases - self._vocabulary_name = name + self._vocabulary_phrases = None + self._vocabulary_name = None self._vocabulary_compiled = False self._vocabulary_path = None + def init(self, name, phrases): + self._vocabulary_phrases = phrases + self._vocabulary_name = name + def compile_vocabulary(self, compilation_func): if self._vocabulary_compiled: raise RuntimeError("Vocabulary has already been compiled!") diff --git a/jasper/pluginstore.py b/jasper/pluginstore.py index 96d1d4478..9d96ef691 100644 --- a/jasper/pluginstore.py +++ b/jasper/pluginstore.py @@ -141,7 +141,8 @@ def __init__(self, plugin_dirs): 'audioengine': plugin.AudioEnginePlugin, 'speechhandler': plugin.SpeechHandlerPlugin, 'tts': plugin.TTSPlugin, - 'stt': plugin.STTPlugin + 'stt': plugin.STTPlugin, + 'tti': plugin.TTIPlugin } def detect_plugins(self): diff --git a/jasper/testutils.py b/jasper/testutils.py index 2bf022f84..253ebb309 100644 --- a/jasper/testutils.py +++ b/jasper/testutils.py @@ -12,6 +12,11 @@ } +class TestTTI(object): + def __init__(self): + pass + + class TestMic(object): def __init__(self, inputs=[]): self.inputs = inputs @@ -31,12 +36,16 @@ def say(self, phrase): self.outputs.append(phrase) -def get_plugin_instance(plugin_class, *extra_args): +def get_genericplugin_instance(plugin_class, *extra_args): info = type('', (object,), { 'name': 'pluginunittest', 'translations': { 'en-US': gettext.NullTranslations() } })() - args = tuple(extra_args) + (info, TEST_PROFILE) + args = (info, TEST_PROFILE)+tuple(extra_args) return plugin_class(*args) + + +def get_plugin_instance(plugin_class, *extra_args): + return get_genericplugin_instance(plugin_class, TestTTI(), TestMic()) diff --git a/plugins/speechhandler/clock/clock.py b/plugins/speechhandler/clock/clock.py index ff2c180f9..0d0d1cada 100644 --- a/plugins/speechhandler/clock/clock.py +++ b/plugins/speechhandler/clock/clock.py @@ -7,6 +7,10 @@ class ClockPlugin(plugin.SpeechHandlerPlugin): def get_phrases(self): return [self.gettext("TIME")] + + def init(self): + SpeechHandlerPlugin.init(self) + self._tti.ACTIONS = [('WHAT TIME IS IT',self.say_time)] def handle(self, text, mic): """ @@ -25,6 +29,23 @@ def handle(self, text, mic): fmt = "It is {t:%l}:{t.minute} {t:%P} right now." mic.say(self.gettext(fmt).format(t=now)) + def say_time(self): + """ + Reports the current time based on the user's timezone. + + Arguments: + text -- user-input, typically transcribed speech + mic -- used to interact with the user (for both input and output) + """ + + tz = app_utils.get_timezone(self.profile) + now = datetime.datetime.now(tz=tz) + if now.minute == 0: + fmt = "It is {t:%l} {t:%P} right now." + else: + fmt = "It is {t:%l}:{t.minute} {t:%P} right now." + mic.say(self.gettext(fmt).format(t=now)) + def is_valid(self, text): """ Returns True if input is related to the time. diff --git a/plugins/stt/julius-stt/julius.py b/plugins/stt/julius-stt/julius.py index cc4457069..c00907646 100644 --- a/plugins/stt/julius-stt/julius.py +++ b/plugins/stt/julius-stt/julius.py @@ -24,6 +24,10 @@ def __init__(self, *args, **kwargs): self._logger.warning("This STT plugin doesn't have multilanguage " + "support!") + + def init(self, *args, **kwargs): + plugin.STTPlugin.init(self, *args, **kwargs) + vocabulary_path = self.compile_vocabulary( juliusvocab.compile_vocabulary) diff --git a/plugins/stt/pocketsphinx-stt/sphinxplugin.py b/plugins/stt/pocketsphinx-stt/sphinxplugin.py index 97e044255..f21e61622 100644 --- a/plugins/stt/pocketsphinx-stt/sphinxplugin.py +++ b/plugins/stt/pocketsphinx-stt/sphinxplugin.py @@ -40,6 +40,9 @@ def __init__(self, *args, **kwargs): self._logger.warning("This STT plugin doesn't have multilanguage " + "support!") + def init(self, *args, **kwargs): + plugin.STTPlugin.init(self, *args, **kwargs) + vocabulary_path = self.compile_vocabulary( sphinxvocab.compile_vocabulary) diff --git a/plugins/stt/pocketsphinx-stt/tests/test_sphinxplugin.py b/plugins/stt/pocketsphinx-stt/tests/test_sphinxplugin.py index a240b9686..90cb1f022 100644 --- a/plugins/stt/pocketsphinx-stt/tests/test_sphinxplugin.py +++ b/plugins/stt/pocketsphinx-stt/tests/test_sphinxplugin.py @@ -12,12 +12,12 @@ def setUp(self): self.time_clip = paths.data('audio', 'time.wav') try: - self.passive_stt_engine = testutils.get_plugin_instance( - sphinxplugin.PocketsphinxSTTPlugin, - 'unittest-passive', ['JASPER']) - self.active_stt_engine = testutils.get_plugin_instance( - sphinxplugin.PocketSphinxSTTPlugin, - 'unittest-active', ['TIME']) + self.passive_stt_engine = testutils.get_genericplugin_instance( + sphinxplugin.PocketsphinxSTTPlugin) + self.passive_stt_engine.init('unittest-passive', ['JASPER']) + self.active_stt_engine = testutils.get_genericplugin_instance( + sphinxplugin.PocketSphinxSTTPlugin) + self.active_stt_engine.init('unittest-active', ['TIME']) except ImportError: self.skipTest("Pockersphinx not installed!") diff --git a/plugins/tti/phrasematcher-tti/__init__.py b/plugins/tti/phrasematcher-tti/__init__.py new file mode 100644 index 000000000..ac6239d85 --- /dev/null +++ b/plugins/tti/phrasematcher-tti/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from .phrasematcher import PhraseMatcherPlugin diff --git a/plugins/tti/phrasematcher-tti/phrasematcher.py b/plugins/tti/phrasematcher-tti/phrasematcher.py new file mode 100644 index 000000000..60a7dde2e --- /dev/null +++ b/plugins/tti/phrasematcher-tti/phrasematcher.py @@ -0,0 +1,71 @@ +#basic implementation taken from https://gist.github.com/Holzhaus/cdf3e9534e6295d40e07 +import itertools +import string +import re +from jasper import plugin + +class PhraseMatcherPlugin(plugin.TTIPlugin): + + def get_phrases(self): + # Sample implementation, there might be a better one + phrases = [] + for base_phrase, action in self.ACTIONS: + placeholders = [x[1] for x in string.Formatter().parse(base_phrase)] + factors = [placeholder_values[placeholder] for placeholder in placeholders] + combinations = itertools.product(*factors) + for combination in combinations: + replacement_values = dict(zip(placeholders,combination)) + phrases.append(base_phrase.format(**replacement_values)) + return phrases + + def get_regex_phrases(self): + return [self.base_phrase_to_regex_pattern(base_phrase) for base_phrase, action in self.ACTIONS] + + def base_phrase_to_regex_pattern(self, base_phrase): + # Sample implementation, I think that this can be improved, too + placeholders = [x[1] for x in string.Formatter().parse(base_phrase)] + placeholder_values = {} + for placeholder in placeholders: + placeholder_values[placeholder] = '(?P<{}>.+)'.format(placeholder) + regex_phrase = "^{}$".format(base_phrase.format(**placeholder_values)) + pattern = re.compile(regex_phrase, re.LOCALE | re.UNICODE) + return pattern + + def match_phrase(self, phrase): + for pattern in self.get_regex_phrases(): + matchobj = pattern.match(phrase) + if matchobj: + return matchobj + return None + + def is_valid(self, phrase): + matchobj = self.match_phrase(phrase) + if matchobj: + return True + return False + + def get_confidence(self, phrase): + return is_valid(self, phrase) + + def get_actionlist(self, phrase): + pass + + def get_intent(self, phrase): + matchobj = self.match_phrase(phrase) + if matchobj: + for base_phrase, action in self.ACTIONS: + if matchobj.re.match(base_phrase): + kwargs = matchobj.groupdict() + #action(**kwargs) + return [action, kwargs] + return [None, None] + + def handle_intent(self, phrase): + matchobj = self.match_phrase(phrase) + if matchobj: + for base_phrase, action in self.ACTIONS: + if matchobj.re.match(base_phrase): + kwargs = matchobj.groupdict() + action(**kwargs) + return True + return False diff --git a/plugins/tti/phrasematcher-tti/plugin.info b/plugins/tti/phrasematcher-tti/plugin.info new file mode 100644 index 000000000..803bad7b3 --- /dev/null +++ b/plugins/tti/phrasematcher-tti/plugin.info @@ -0,0 +1,10 @@ +[Plugin] +Name = phrasematcher-tti +Version = 1.0.0 +License = MIT +URL = http://jasperproject.github.io/ +Description = Text-to-intent matching which relies on phrase matching using regex. + +[Author] +Name = Jasper Project +URL = http://jasperproject.github.io/ diff --git a/plugins/tti/phrasematcher-tti/test_phrasematcher.py b/plugins/tti/phrasematcher-tti/test_phrasematcher.py new file mode 100644 index 000000000..8e675d833 --- /dev/null +++ b/plugins/tti/phrasematcher-tti/test_phrasematcher.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +import unittest +import time +from jasper import testutils +from . import phrasematcher + +# +# Some sample input +# + +sample_phrases = [("LightController", "SWITCH LIVINGROOM LIGHTS TO COLOR RED"), + ("Clock", "WHAT TIME IS IT"), + ("None", "THIS IS A TEST PHRASE NO PLUGIN UNDERSTANDS"), + ("LightController", "SWITCH LIGHTS OFF")] + +class LightController(): + + def __init__(self, mic): + self._mic = mic + self._tti=testutils.get_genericplugin_instance(phrasematcher.PhraseMatcherPlugin) + self._tti.WORDS = {'location': ['ALL', 'BEDROOM', 'LIVINGROOM','BATHROOM'], + 'color': ['BLUE','YELLOW','RED', 'GREEN'], + 'state': ['ON','OFF']} + self._tti.ACTIONS = [('SWITCH {location} LIGHTS TO COLOR {color}', LightController.change_light_color), + ('SWITCH LIGHTS {state}', LightController.switch_light_state)] + + def change_light_color(self, **kwargs): + self._mic.say("Changing {location} light colors to {color} now...".format(**kwargs)) + + def switch_light_state(self, **kwargs): + self._mic.say("Switching lights {state} now...".format(**kwargs)) + + def handle(self, phrase): + action, param = self._tti.get_intent(phrase) + if action: + action(self, **param) + + def is_valid(self, phrase): + self._tti.is_valid(phrase) + + def name(self): + return "LightController" + +class Clock(): + + def __init__(self, mic): + self._mic = mic + self._tti=testutils.get_genericplugin_instance(phrasematcher.PhraseMatcherPlugin) + self._tti.ACTIONS = [('WHAT TIME IS IT',Clock.say_time)]# + + def say_time(self, **kwargs): + self._mic.say(time.strftime("The time is %H hours and %M minutes.")) + + def handle(self, phrase): + action, param = self._tti.get_intent(phrase) + if action: + action(self, **param) + + def is_valid(self, phrase): + self._tti.is_valid(phrase) + + def name(self): + return "Clock" + +class TestPhrasematcherPlugin(unittest.TestCase): + def setUp(self): + self.mic = testutils.TestMic() + self.plugins = [LightController(self.mic), Clock(self.mic)] + + def handle(self, phrase): + handled = False + for plugin in self.plugins: + if plugin.handle(phrase): + handled = True + return handled + + def test_is_valid_method(self): + for plugin in self.plugins: + for pluginname, phrase in sample_phrases: + if pluginname == plugin.name: + self.assertTrue(plugin.is_valid(phrase)) + else: + self.assertFalse(plugin.is_valid(phrase)) + + def test_handle_method(self): + for pluginname, phrase in sample_phrases: + self.handle(phrase) + self.assertEqual(len(self.mic.outputs), 3) + #print self.mic.outputs diff --git a/tests/test_brain.py b/tests/test_brain.py index 41b4785df..2ccfa98f1 100644 --- a/tests/test_brain.py +++ b/tests/test_brain.py @@ -24,8 +24,8 @@ def is_valid(self, text): class TestBrain(unittest.TestCase): def testPriority(self): - """Does Brain sort modules by priority?""" - my_brain = brain.Brain(testutils.TEST_PROFILE) + """Does Brain sort modules by priority?""" + my_brain = brain.Brain(testutils.TEST_PROFILE, ExamplePlugin(['MOCK_TTI'])) plugin1 = ExamplePlugin(['MOCK1'], priority=1) plugin2 = ExamplePlugin(['MOCK1'], priority=999) @@ -52,7 +52,7 @@ def testPriority(self): def testPluginPhraseExtraction(self): expected_phrases = ['MOCK1', 'MOCK2'] - my_brain = brain.Brain(testutils.TEST_PROFILE) + my_brain = brain.Brain(testutils.TEST_PROFILE, ExamplePlugin(['MOCK_TTI'])) my_brain.add_plugin(ExamplePlugin(['MOCK2'])) my_brain.add_plugin(ExamplePlugin(['MOCK1'])) @@ -64,7 +64,7 @@ def testPluginPhraseExtraction(self): def testStandardPhraseExtraction(self): expected_phrases = ['MOCK'] - my_brain = brain.Brain(testutils.TEST_PROFILE) + my_brain = brain.Brain(testutils.TEST_PROFILE, ExamplePlugin(['MOCK_TTI'])) with tempfile.TemporaryFile() as f: # We can't use mock_open here, because it doesn't seem to work