From 30e3c8fa9a53a044e92c69881d1ee2f75c5bbbe9 Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 19 Mar 2021 15:05:28 +0100 Subject: [PATCH 1/8] Pass interim and final transcripts to onResult callback Inspired by https://developers.google.com/web/updates/2013/01/Voice-Driven-Web-Apps-Introduction-to-the-Web-Speech-API --- src/useSpeechRecognition.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/useSpeechRecognition.js b/src/useSpeechRecognition.js index ac3695a..0dc3f88 100644 --- a/src/useSpeechRecognition.js +++ b/src/useSpeechRecognition.js @@ -41,7 +41,17 @@ const useSpeechRecognition = (props = {}) => { .map((result) => result.transcript) .join(''); - onResult(transcript); + let final = ''; + let interim = ''; + for (let i = event.resultIndex; i < event.results.length; ++i) { + if (event.results[i].isFinal) { + final = `${final}${event.results[i][0].transcript}`; + } else { + interim = `${interim}${event.results[i][0].transcript}`; + } + } + + onResult(transcript, final, interim); }; const handleError = (event) => { From 6e7e87870d4711a94ba9614c4f6d4db500043c04 Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 19 Mar 2021 17:21:19 +0100 Subject: [PATCH 2/8] Add config to let SpeechRecognition stop automatically after inactivity --- README.md | 12 +- .../src/useSpeechRecognitionContinuous.jsx | 118 ++++++++++++++++++ src/useSpeechRecognition.js | 10 +- 3 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 examples/src/useSpeechRecognitionContinuous.jsx diff --git a/README.md b/README.md index 48b3eba..437f680 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,9 @@ Called when SpeechRecognition stops listening. #### onResult -`function(string)` +`function(transcript: string, final: string, interim: string)` -Called when SpeechRecognition has a result. It is called with a string containing a transcript of the recognized speech. +Called when SpeechRecognition has a result. It is called with a string containing a transcript of the entire recognized speech, a string containing the final transcript of the most recent recognized speech, and a string containing the interim transcript of current speech. ### Returns @@ -171,6 +171,14 @@ Call to make the browser start listening for input. Every time it processes a re `boolean` _(default: true)_ SpeechRecognition can provide realtime results as it's trying to figure out the best match for the input. Set to false if you only want the final result. +- **continuous** + `boolean` _(default: false)_ + The continuous property of the SpeechRecognition interface controls whether continuous results are returned for each recognition, or only a single result. + +- **nonStop** + `boolean` _(default: true)_ + When set to `true` SpeechRecognition will not stop automatically after inactivity. + #### stop `function()` diff --git a/examples/src/useSpeechRecognitionContinuous.jsx b/examples/src/useSpeechRecognitionContinuous.jsx new file mode 100644 index 0000000..7722cbf --- /dev/null +++ b/examples/src/useSpeechRecognitionContinuous.jsx @@ -0,0 +1,118 @@ +import React, { useState } from 'react'; +import { useSpeechRecognition } from '../../src'; +import { Container } from './shared'; + +const languageOptions = [ + { label: 'Cambodian', value: 'km-KH' }, + { label: 'Deutsch', value: 'de-DE' }, + { label: 'English', value: 'en-AU' }, + { label: 'Farsi', value: 'fa-IR' }, + { label: 'Français', value: 'fr-FR' }, + { label: 'Italiano', value: 'it-IT' }, + { label: '普通话 (中国大陆) - Mandarin', value: 'zh' }, + { label: 'Portuguese', value: 'pt-BR' }, + { label: 'Español', value: 'es-MX' }, + { label: 'Svenska - Swedish', value: 'sv-SE' }, +]; + +const Example = () => { + const [lang, setLang] = useState('en-AU'); + const [final, setFinal] = useState(''); + const [interim, setInterim] = useState(''); + const [blocked, setBlocked] = useState(false); + + const onEnd = () => { + console.log('onEnd()'); + setFinal(prevState => `${prevState}${interim} `); + setInterim(''); + }; + + const onResult = (_, finalTranscript, interimTranscript) => { + console.log('finalTranscript', finalTranscript); + console.log('interimTranscript', interimTranscript); + // setInterim(''); + setInterim(interimTranscript); + setFinal(prevState => `${prevState}${finalTranscript}`); + // setInterim(prevState => `${prevState}${interimTranscript}`); + }; + + const changeLang = (event) => { + setLang(event.target.value); + }; + + const onError = (event) => { + if (event.error === 'not-allowed') { + setBlocked(true); + } + }; + + const { listen, listening, stop, supported } = useSpeechRecognition({ + onResult, + onEnd, + onError, + }); + + const toggle = listening + ? stop + : () => { + setBlocked(false); + listen({ + continuous: true, + nonStop: false, + lang, + }); + }; + + return ( + +
+

Continuous Recognition

+ {!supported && ( +

+ Oh no, it looks like your browser doesn't support Speech + Recognition. +

+ )} + {supported && ( + +

+ {`Click 'Listen' and start speaking. + SpeechRecognition will provide a transcript of what you are saying.`} +

+ + + +
+ {final && {final}} + {interim && {interim}} +
+ + {blocked && ( +

+ The microphone is blocked for this site in your browser. +

+ )} + + )} +
+ + ); +}; + +export default Example; diff --git a/src/useSpeechRecognition.js b/src/useSpeechRecognition.js index ac3695a..1c9fa43 100644 --- a/src/useSpeechRecognition.js +++ b/src/useSpeechRecognition.js @@ -60,6 +60,7 @@ const useSpeechRecognition = (props = {}) => { continuous = false, maxAlternatives = 1, grammars, + nonStop = true, } = args; setListening(true); recognition.current.lang = lang; @@ -73,7 +74,14 @@ const useSpeechRecognition = (props = {}) => { } // SpeechRecognition stops automatically after inactivity // We want it to keep going until we tell it to stop - recognition.current.onend = () => recognition.current.start(); + recognition.current.onend = () => { + setListening(false); + onEnd(); + if (nonStop) { + setListening(true); + recognition.current.start(); + } + } recognition.current.start(); }, [listening, supported, recognition]); From 66cc598dfbe7b0fce45272ff356e0cff649b4e70 Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 19 Mar 2021 17:23:18 +0100 Subject: [PATCH 3/8] Continuous speech example, with css styles --- examples/src/index.jsx | 2 ++ examples/src/shared.js | 23 +++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/examples/src/index.jsx b/examples/src/index.jsx index 17cbf34..433a160 100644 --- a/examples/src/index.jsx +++ b/examples/src/index.jsx @@ -2,6 +2,7 @@ import React from 'react'; import { render } from 'react-dom'; import SpeechSynthesisExample from './useSpeechSynthesis'; import SpeechRecognitionExample from './useSpeechRecognition'; +import SpeechRecognitionContinuousExample from './useSpeechRecognitionContinuous'; import { GlobalStyles, Row, GitLink, Title } from './shared'; import gh from './images/github.png'; @@ -17,6 +18,7 @@ const App = () => ( + Github diff --git a/examples/src/shared.js b/examples/src/shared.js index 67ae730..61be6b8 100644 --- a/examples/src/shared.js +++ b/examples/src/shared.js @@ -58,16 +58,35 @@ export const Container = styled.div` } select, - textarea { + textarea, + .textarea { font-size: 16px; margin-bottom: 12px; width: 100%; } - textarea { + textarea, + .textarea { border: 1px solid darkgrey; border-radius: 10px; padding: 8px; resize: none; } + + .textarea { + box-sizing: border-box; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + min-height: 72px; + background-color: white; + font-family: monospace; + } + + .final { + color: #666; + } + + .interim { + color: black; + } `; From 150e549ea4011399bc3dfa350f657b355e77bfac Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 19 Mar 2021 17:47:19 +0100 Subject: [PATCH 4/8] Remove comments --- examples/src/useSpeechRecognitionContinuous.jsx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/examples/src/useSpeechRecognitionContinuous.jsx b/examples/src/useSpeechRecognitionContinuous.jsx index 7722cbf..a6abd72 100644 --- a/examples/src/useSpeechRecognitionContinuous.jsx +++ b/examples/src/useSpeechRecognitionContinuous.jsx @@ -22,18 +22,13 @@ const Example = () => { const [blocked, setBlocked] = useState(false); const onEnd = () => { - console.log('onEnd()'); setFinal(prevState => `${prevState}${interim} `); setInterim(''); }; const onResult = (_, finalTranscript, interimTranscript) => { - console.log('finalTranscript', finalTranscript); - console.log('interimTranscript', interimTranscript); - // setInterim(''); setInterim(interimTranscript); setFinal(prevState => `${prevState}${finalTranscript}`); - // setInterim(prevState => `${prevState}${interimTranscript}`); }; const changeLang = (event) => { From 4d2dc1e07c2676973a85b4ffa72d372977183351 Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 19 Mar 2021 17:48:09 +0100 Subject: [PATCH 5/8] Fix HTML element IDs --- examples/src/useSpeechRecognitionContinuous.jsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/examples/src/useSpeechRecognitionContinuous.jsx b/examples/src/useSpeechRecognitionContinuous.jsx index a6abd72..31abca9 100644 --- a/examples/src/useSpeechRecognitionContinuous.jsx +++ b/examples/src/useSpeechRecognitionContinuous.jsx @@ -60,7 +60,7 @@ const Example = () => { return ( -
+

Continuous Recognition

{!supported && (

@@ -74,10 +74,10 @@ const Example = () => { {`Click 'Listen' and start speaking. SpeechRecognition will provide a transcript of what you are saying.`}

- + - -
+ +
{final && {final}} {interim && {interim}}
From 66c386eeb159b942b9378f543833b1c16ab6d273 Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 21 Mar 2021 03:30:10 +0100 Subject: [PATCH 6/8] declarative rather than Imperative programming --- src/useSpeechRecognition.js | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/useSpeechRecognition.js b/src/useSpeechRecognition.js index f359e2c..ee32c5c 100644 --- a/src/useSpeechRecognition.js +++ b/src/useSpeechRecognition.js @@ -36,20 +36,23 @@ const useSpeechRecognition = (props = {}) => { const [supported, setSupported] = useState(false); const processResult = (event) => { - const transcript = Array.from(event.results) - .map((result) => result[0]) - .map((result) => result.transcript) - .join(''); + const concat = (arr) => ( + arr.map((result) => result[0]) + .map((result) => result.transcript) + .join('') + ); - let final = ''; - let interim = ''; - for (let i = event.resultIndex; i < event.results.length; ++i) { - if (event.results[i].isFinal) { - final = `${final}${event.results[i][0].transcript}`; - } else { - interim = `${interim}${event.results[i][0].transcript}`; - } - } + const transcript = concat(Array.from(event.results)); + + const results = Array.from(event.results) + .splice(event.resultIndex); + + const final = concat( + results.filter((result) => result.isFinal) + ); + const interim = concat( + results.filter((result) => !result.isFinal) + ); onResult(transcript, final, interim); }; From 0d4032e0e7a8959c24bcb6ef542be27bbd66ce21 Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 21 Mar 2021 04:08:07 +0100 Subject: [PATCH 7/8] DRY --- src/useSpeechRecognition.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/useSpeechRecognition.js b/src/useSpeechRecognition.js index ee32c5c..d658fdc 100644 --- a/src/useSpeechRecognition.js +++ b/src/useSpeechRecognition.js @@ -42,16 +42,15 @@ const useSpeechRecognition = (props = {}) => { .join('') ); - const transcript = concat(Array.from(event.results)); - - const results = Array.from(event.results) - .splice(event.resultIndex); + const results = Array.from(event.results); + const spliced = results.splice(event.resultIndex) + const transcript = concat(results); const final = concat( - results.filter((result) => result.isFinal) + spliced.filter((result) => result.isFinal) ); const interim = concat( - results.filter((result) => !result.isFinal) + spliced.filter((result) => !result.isFinal) ); onResult(transcript, final, interim); From a0d3e529e6e6bd6ad1ed23230e7f7cf8317300dd Mon Sep 17 00:00:00 2001 From: Matthew Date: Sun, 21 Mar 2021 11:35:03 +0100 Subject: [PATCH 8/8] improve nomenclature --- src/useSpeechRecognition.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/useSpeechRecognition.js b/src/useSpeechRecognition.js index d658fdc..bf0737d 100644 --- a/src/useSpeechRecognition.js +++ b/src/useSpeechRecognition.js @@ -36,21 +36,21 @@ const useSpeechRecognition = (props = {}) => { const [supported, setSupported] = useState(false); const processResult = (event) => { - const concat = (arr) => ( + const merge = (arr) => ( arr.map((result) => result[0]) .map((result) => result.transcript) .join('') ); const results = Array.from(event.results); - const spliced = results.splice(event.resultIndex) + const changed = results.splice(event.resultIndex) - const transcript = concat(results); - const final = concat( - spliced.filter((result) => result.isFinal) + const transcript = merge(results); + const final = merge( + changed.filter((result) => result.isFinal) ); - const interim = concat( - spliced.filter((result) => !result.isFinal) + const interim = merge( + changed.filter((result) => !result.isFinal) ); onResult(transcript, final, interim);