From 8fb147ceb37c2fa6db74d04c1aba23efcf90b5d4 Mon Sep 17 00:00:00 2001 From: Henri Woodcock Date: Fri, 21 May 2021 19:10:17 +0100 Subject: [PATCH] added pdm and moved analog --- CMakeLists.txt | 112 ++++++++++++-- ...o_provider.cc => analog_audio_provider.cc} | 0 src/pdm_audio_provider.cc | 139 ++++++++++++++++++ 3 files changed, 242 insertions(+), 9 deletions(-) rename src/{audio_provider.cc => analog_audio_provider.cc} (100%) create mode 100644 src/pdm_audio_provider.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ed0e63..6b2b950 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,17 +15,17 @@ set(CMAKE_CXX_STANDARD 11) # initialize the Pico SDK pico_sdk_init() -add_executable(pico_micro_speech "") +add_executable(pico_micro_speech_analog "") set(PICO_TFLMICRO_MICRO_SPEECH_PATH ${CMAKE_CURRENT_LIST_DIR}/lib/pico-tflmicro/examples/micro_speech) -target_include_directories(pico_micro_speech +target_include_directories(pico_micro_speech_analog PRIVATE ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/. ) set_target_properties( - pico_micro_speech + pico_micro_speech_analog PROPERTIES COMPILE_FLAGS -fno-rtti COMPILE_FLAGS -fno-exceptions @@ -33,7 +33,7 @@ set_target_properties( COMPILE_FLAGS -nostdlib ) -target_sources(pico_micro_speech +target_sources(pico_micro_speech_analog PRIVATE ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/fft.cpp ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/fft_util.cpp @@ -50,7 +50,7 @@ target_sources(pico_micro_speech ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/window.c ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/window_util.c - ${CMAKE_CURRENT_LIST_DIR}/src/audio_provider.cc + ${CMAKE_CURRENT_LIST_DIR}/src/analog_audio_provider.cc ${CMAKE_CURRENT_LIST_DIR}/src/command_responder.cc ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/feature_provider.cpp ${CMAKE_CURRENT_LIST_DIR}/src/main.cpp @@ -93,7 +93,7 @@ target_sources(pico_micro_speech ) target_link_libraries( - pico_micro_speech + pico_micro_speech_analog pico-tflmicro hardware_adc hardware_dma @@ -101,11 +101,105 @@ target_link_libraries( pico_analog_microphone ) -pico_add_extra_outputs(pico_micro_speech) +pico_add_extra_outputs(pico_micro_speech_analog) # enable usb output, disable uart output -pico_enable_stdio_usb(pico_micro_speech 1) -pico_enable_stdio_uart(pico_micro_speech 0) +pico_enable_stdio_usb(pico_micro_speech_analog 1) +pico_enable_stdio_uart(pico_micro_speech_analog 0) + + +add_executable(pico_micro_speech_pdm "") + +set(PICO_TFLMICRO_MICRO_SPEECH_PATH ${CMAKE_CURRENT_LIST_DIR}/lib/pico-tflmicro/examples/micro_speech) + +target_include_directories(pico_micro_speech_pdm + PRIVATE + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/. +) + +set_target_properties( + pico_micro_speech_pdm + PROPERTIES + COMPILE_FLAGS -fno-rtti + COMPILE_FLAGS -fno-exceptions + COMPILE_FLAGS -fno-threadsafe-statics + COMPILE_FLAGS -nostdlib +) + +target_sources(pico_micro_speech_pdm + PRIVATE + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/fft.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/fft_util.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/filterbank.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/filterbank_util.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/frontend.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/frontend_util.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/log_lut.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/log_scale.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/log_scale_util.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/noise_reduction.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/window.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/window_util.c + ${CMAKE_CURRENT_LIST_DIR}/src/pdm_audio_provider.cc + ${CMAKE_CURRENT_LIST_DIR}/src/command_responder.cc + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/feature_provider.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/main.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/main_functions.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/micro_features_generator.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/micro_model_settings.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/model.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/recognize_commands.cpp + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/micro/tools/make/downloads/kissfft/tools/kiss_fftr.c + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/bits.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/fft.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/fft_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/filterbank.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/filterbank_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/frontend.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/frontend_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/log_lut.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/log_scale.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/log_scale_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/noise_reduction.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/noise_reduction_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/pcan_gain_control_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/window.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/experimental/microfrontend/lib/window_util.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/audio_provider.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/command_responder.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/feature_provider.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/main_functions.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/micro_features_generator.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/micro_features_generator.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/micro_model_settings.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/micro_features/model.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/recognize_commands.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/micro/tools/make/downloads/kissfft/COPYING + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/micro/tools/make/downloads/kissfft/_kiss_fft_guts.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.h + ${PICO_TFLMICRO_MICRO_SPEECH_PATH}/tensorflow/lite/micro/tools/make/downloads/kissfft/tools/kiss_fftr.h +) + +target_link_libraries( + pico_micro_speech_pdm + pico-tflmicro + hardware_adc + hardware_dma + hardware_pwm + hardware_pio + pico_pdm_microphone +) + +pico_add_extra_outputs(pico_micro_speech_pdm) + +# enable usb output, disable uart output +pico_enable_stdio_usb(pico_micro_speech_pdm 1) +pico_enable_stdio_uart(pico_micro_speech_pdm 0) add_subdirectory("lib/pico-tflmicro" EXCLUDE_FROM_ALL) add_subdirectory("lib/pico-microphone" EXCLUDE_FROM_ALL) diff --git a/src/audio_provider.cc b/src/analog_audio_provider.cc similarity index 100% rename from src/audio_provider.cc rename to src/analog_audio_provider.cc diff --git a/src/pdm_audio_provider.cc b/src/pdm_audio_provider.cc new file mode 100644 index 0000000..d9dfdd5 --- /dev/null +++ b/src/pdm_audio_provider.cc @@ -0,0 +1,139 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "audio_provider.h" + +extern "C" { +#include "pico/pdm_microphone.h" +} + +#include "micro_features/micro_model_settings.h" + +#define DEFAULT_PDM_BUFFER_SIZE 256 + +namespace { +bool g_is_audio_initialized = false; +// An internal buffer able to fit 16x our sample size +constexpr int kAudioCaptureBufferSize = DEFAULT_PDM_BUFFER_SIZE * 16; +int16_t g_audio_capture_buffer[kAudioCaptureBufferSize]; +// A buffer that holds our output +int16_t g_audio_output_buffer[kMaxAudioSampleSize]; +// Mark as volatile so we can check in a while loop to see if +// any samples have arrived yet. +volatile int32_t g_latest_audio_timestamp = 0; + +struct pdm_microphone_config pdm_config = { + .gpio_data = 2, + .gpio_clk = 3, + .pio = pio0, + .pio_sm = 0, + .sample_rate = 16000, + .sample_buffer_size = 256, +}; + +// struct analog_microphone_config config = { +// .gpio = 26, +// .bias_voltage = 1.25, +// .sample_rate = 16000, +// .sample_buffer_size = 256, +// }; + + +} // namespace + +#include "pico/stdlib.h" + +void CaptureSamples() { + // This is how many bytes of new data we have each time this is called + const int number_of_samples = DEFAULT_PDM_BUFFER_SIZE; + // Calculate what timestamp the last audio sample represents + const int32_t time_in_ms = + g_latest_audio_timestamp + + (number_of_samples / (kAudioSampleFrequency / 1000)); + // Determine the index, in the history of all samples, of the last sample + const int32_t start_sample_offset = + g_latest_audio_timestamp * (kAudioSampleFrequency / 1000); + // Determine the index of this sample in our ring buffer + const int capture_index = start_sample_offset % kAudioCaptureBufferSize; + // Read the data to the correct place in our buffer + pdm_microphone_read(g_audio_capture_buffer + capture_index, DEFAULT_PDM_BUFFER_SIZE); + // analog_microphone_read(g_audio_capture_buffer + capture_index, DEFAULT_PDM_BUFFER_SIZE); + // This is how we let the outside world know that new audio data has arrived. + g_latest_audio_timestamp = time_in_ms; + + // uint32_t now = to_ms_since_boot(get_absolute_time()); + // printf("now = %u\n", now); +} + +TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) { + pdm_microphone_init(&pdm_config); + // pdm_microphone_set_filter_gain(20); + // pdm_microphone_set_filter_max_volume(128); + pdm_microphone_set_samples_ready_handler(CaptureSamples); + pdm_microphone_start(); + + // analog_microphone_init(&config); + // analog_microphone_set_samples_ready_handler(CaptureSamples); + // analog_microphone_start(); + + // Block until we have our first audio sample + while (!g_latest_audio_timestamp) { + } + + return kTfLiteOk; +} + +TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter, + int start_ms, int duration_ms, + int* audio_samples_size, int16_t** audio_samples) { + // Set everything up to start receiving audio + if (!g_is_audio_initialized) { + TfLiteStatus init_status = InitAudioRecording(error_reporter); + if (init_status != kTfLiteOk) { + return init_status; + } + g_is_audio_initialized = true; + } + // This next part should only be called when the main thread notices that the + // latest audio sample data timestamp has changed, so that there's new data + // in the capture ring buffer. The ring buffer will eventually wrap around and + // overwrite the data, but the assumption is that the main thread is checking + // often enough and the buffer is large enough that this call will be made + // before that happens. + + // Determine the index, in the history of all samples, of the first + // sample we want + const int start_offset = start_ms * (kAudioSampleFrequency / 1000); + // Determine how many samples we want in total + const int duration_sample_count = + duration_ms * (kAudioSampleFrequency / 1000); + for (int i = 0; i < duration_sample_count; ++i) { + // For each sample, transform its index in the history of all samples into + // its index in g_audio_capture_buffer + const int capture_index = (start_offset + i) % kAudioCaptureBufferSize; + // Write the sample to the output buffer + g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index]; + } + + // Set pointers to provide access to the audio + *audio_samples_size = kMaxAudioSampleSize; + *audio_samples = g_audio_output_buffer; + + return kTfLiteOk; +} + +int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }