From 981e45f5ad76386c281af6018efbeb7ea84d7325 Mon Sep 17 00:00:00 2001 From: Charity Loh <55676809+charitylxy@users.noreply.github.com> Date: Mon, 4 Mar 2024 22:47:15 +0800 Subject: [PATCH] Support Non-ASCII Characters for User-Provided String (#518) * Convert Parameter Strings to UTF-8 * update dll and decode values with utf-8 * address comments * address comments * fix typo * Add automatic fallback and feature toggle * address comments * address comments * remove .env temporary * address comment * add codegen changes * add .env to gitignore * minor fix * fix indentation * modify encoding for missing functions --- .env.sample | 14 ++++ .gitignore | 1 + generated/nidaqmx/_lib.py | 67 ++++++++++++++----- generated/nidaqmx/_library_interpreter.py | 64 +++++++++--------- .../templates/_library_interpreter.py.mako | 4 +- src/codegen/utilities/interpreter_helpers.py | 4 +- src/handwritten/_lib.py | 67 ++++++++++++++----- 7 files changed, 155 insertions(+), 66 deletions(-) create mode 100644 .env.sample diff --git a/.env.sample b/.env.sample new file mode 100644 index 00000000..024c23c8 --- /dev/null +++ b/.env.sample @@ -0,0 +1,14 @@ +# This is a sample nidaqmx-python configuration file. + +# To use it: +# - Copy this file to your application's directory or one of its parent directories +# (such as the root of your Git repository). +# - Rename it to `.env`. +# - Uncomment and edit the options you want to change. +# - Restart any affected applications or services. + +# By default, nidaqmx-python on Windows uses nicai_utf8, the UTF-8 version of the NI-DAQmx C library. +# If that is not available, it falls back to nicaiu, the MBCS (multibyte character set) version. You can override +# this behavior by uncommenting the following option: + +# NIDAQMX_C_LIBRARY=nicaiu \ No newline at end of file diff --git a/.gitignore b/.gitignore index c71a6d43..af03b36c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ test_results/ htmlcov/ # Environments +.env .venv # Built artifacts diff --git a/generated/nidaqmx/_lib.py b/generated/nidaqmx/_lib.py index 33b9e21b..6789d402 100644 --- a/generated/nidaqmx/_lib.py +++ b/generated/nidaqmx/_lib.py @@ -6,6 +6,8 @@ import platform import sys import threading +import locale +from decouple import config from typing import cast, TYPE_CHECKING from nidaqmx.errors import DaqNotFoundError, DaqNotSupportedError, DaqFunctionNotSupportedError @@ -46,13 +48,13 @@ def _setter(self, val): class CtypesByteString: """ - Custom argtype that automatically converts unicode strings to ASCII - strings in Python 3. + Custom argtype that automatically converts unicode strings to encoding + used by the DAQmx C API DLL in Python 3. """ @classmethod def from_param(cls, param): if isinstance(param, str): - param = param.encode('ascii') + param = param.encode(lib_importer.encoding) return ctypes.c_char_p(param) @@ -128,6 +130,7 @@ def __init__(self): self._cdll = None self._cal_handle = None self._task_handle = None + self._encoding = None @property def windll(self): @@ -148,32 +151,65 @@ def task_handle(self) -> type: @property def cal_handle(self) -> type: return CalHandle - + + @property + def encoding(self): + if self._encoding is None: + self._import_lib() + return self._encoding + def _import_lib(self): """ Determines the location of and loads the NI-DAQmx CAI DLL. """ self._windll = None self._cdll = None + self._encoding = None windll = None cdll = None - - if sys.platform.startswith('win') or sys.platform.startswith('cli'): - try: - if 'iron' in platform.python_implementation().lower(): - windll = ctypes.windll.nicaiu - cdll = ctypes.cdll.nicaiu - else: - windll = ctypes.windll.LoadLibrary('nicaiu') - cdll = ctypes.cdll.LoadLibrary('nicaiu') - except (OSError, WindowsError) as e: - raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) from e + encoding = None + + if sys.platform.startswith('win'): + + def _load_lib(libname: str): + windll = ctypes.windll.LoadLibrary(libname) + cdll = ctypes.cdll.LoadLibrary(libname) + return windll, cdll + + # Feature Toggle to load nicaiu.dll or nicai_utf8.dll + # The Feature Toggle can be set in the .env file + nidaqmx_c_library = config('NIDAQMX_C_LIBRARY', default=None) + + if nidaqmx_c_library is not None: + try: + if nidaqmx_c_library=="nicaiu": + windll, cdll = _load_lib("nicaiu") + encoding = locale.getlocale()[1] + elif nidaqmx_c_library=="nicai_utf8": + windll, cdll = _load_lib("nicai_utf8") + encoding = 'utf-8' + else: + raise ValueError(f"Unsupported NIDAQMX_C_LIBRARY value: {nidaqmx_c_library}") + except (OSError, WindowsError) as e: + raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) from e + else: + try: + windll, cdll = _load_lib("nicai_utf8") + encoding = 'utf-8' + except (OSError, WindowsError): + # Fallback to nicaiu.dll if nicai_utf8.dll cannot be loaded + try: + windll, cdll = _load_lib("nicaiu") + encoding = locale.getlocale()[1] + except (OSError, WindowsError) as e: + raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) from e elif sys.platform.startswith('linux'): # On linux you can use the command find_library('nidaqmx') if find_library('nidaqmx') is not None: cdll = ctypes.cdll.LoadLibrary(find_library('nidaqmx')) windll = cdll + encoding = locale.getlocale()[1] else: raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) else: @@ -181,6 +217,7 @@ def _import_lib(self): self._windll = DaqFunctionImporter(windll) self._cdll = DaqFunctionImporter(cdll) + self._encoding = encoding lib_importer = DaqLibImporter() diff --git a/generated/nidaqmx/_library_interpreter.py b/generated/nidaqmx/_library_interpreter.py index c40977e2..b83cbfc9 100644 --- a/generated/nidaqmx/_library_interpreter.py +++ b/generated/nidaqmx/_library_interpreter.py @@ -94,7 +94,7 @@ def add_network_device( else: break self.check_for_error(size_or_code) - return device_name_out.value.decode('ascii') + return device_name_out.value.decode(lib_importer.encoding) def are_configured_cdaq_sync_ports_disconnected( self, chassis_devices_ports, timeout): @@ -2322,7 +2322,7 @@ def get_auto_configured_cdaq_sync_connections(self): else: break self.check_for_error(size_or_code) - return port_list.value.decode('ascii') + return port_list.value.decode(lib_importer.encoding) def get_buffer_attribute_uint32(self, task, attribute): value = ctypes.c_uint32() @@ -2391,7 +2391,7 @@ def get_cal_info_attribute_string(self, device_name, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_cal_info_attribute_uint32(self, device_name, attribute): value = ctypes.c_uint32() @@ -2505,7 +2505,7 @@ def get_chan_attribute_string(self, task, channel, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_chan_attribute_uint32(self, task, channel, attribute): value = ctypes.c_uint32() @@ -2640,7 +2640,7 @@ def get_device_attribute_string(self, device_name, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_device_attribute_uint32(self, device_name, attribute): value = ctypes.c_uint32() @@ -2771,7 +2771,7 @@ def get_disconnected_cdaq_sync_ports(self): else: break self.check_for_error(size_or_code) - return port_list.value.decode('ascii') + return port_list.value.decode(lib_importer.encoding) def get_exported_signal_attribute_bool(self, task, attribute): value = c_bool32() @@ -2840,7 +2840,7 @@ def get_exported_signal_attribute_string(self, task, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_exported_signal_attribute_uint32(self, task, attribute): value = ctypes.c_uint32() @@ -2918,7 +2918,7 @@ def get_persisted_chan_attribute_string(self, channel, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_persisted_scale_attribute_bool(self, scale_name, attribute): value = c_bool32() @@ -2957,7 +2957,7 @@ def get_persisted_scale_attribute_string(self, scale_name, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_persisted_task_attribute_bool(self, task_name, attribute): value = c_bool32() @@ -2996,7 +2996,7 @@ def get_persisted_task_attribute_string(self, task_name, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_physical_chan_attribute_bool(self, physical_channel, attribute): value = c_bool32() @@ -3142,7 +3142,7 @@ def get_physical_chan_attribute_string(self, physical_channel, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_physical_chan_attribute_uint32(self, physical_channel, attribute): value = ctypes.c_uint32() @@ -3252,7 +3252,7 @@ def get_read_attribute_string(self, task, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_read_attribute_uint32(self, task, attribute): value = ctypes.c_uint32() @@ -3361,7 +3361,7 @@ def get_scale_attribute_string(self, scale_name, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_self_cal_last_date_and_time(self, device_name): year = ctypes.c_uint() @@ -3409,7 +3409,7 @@ def get_system_info_attribute_string(self, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_system_info_attribute_uint32(self, attribute): value = ctypes.c_uint32() @@ -3463,7 +3463,7 @@ def get_task_attribute_string(self, task, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_task_attribute_uint32(self, task, attribute): value = ctypes.c_uint32() @@ -3581,7 +3581,7 @@ def get_timing_attribute_ex_string(self, task, device_names, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_timing_attribute_ex_uint32(self, task, device_names, attribute): value = ctypes.c_uint32() @@ -3652,7 +3652,7 @@ def get_timing_attribute_string(self, task, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_timing_attribute_uint32(self, task, attribute): value = ctypes.c_uint32() @@ -3801,7 +3801,7 @@ def get_trig_attribute_string(self, task, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_trig_attribute_timestamp(self, task, attribute): value = AbsoluteTime() @@ -3904,7 +3904,7 @@ def get_watchdog_attribute_string(self, task, lines, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_write_attribute_bool(self, task, attribute): value = c_bool32() @@ -3973,7 +3973,7 @@ def get_write_attribute_string(self, task, attribute): else: break self.check_for_error(size_or_code) - return value.value.decode('ascii') + return value.value.decode(lib_importer.encoding) def get_write_attribute_uint32(self, task, attribute): value = ctypes.c_uint32() @@ -4980,7 +4980,7 @@ def set_cal_info_attribute_string(self, device_name, attribute, value): ctypes_byte_str, ctypes.c_int32] error_code = cfunc( - device_name, attribute, value.encode('ascii')) + device_name, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_cal_info_attribute_uint32(self, device_name, attribute, value): @@ -5058,7 +5058,7 @@ def set_chan_attribute_string(self, task, channel, attribute, value): ctypes.c_int32] error_code = cfunc( - task, channel, attribute, value.encode('ascii')) + task, channel, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_chan_attribute_uint32(self, task, channel, attribute, value): @@ -5175,7 +5175,7 @@ def set_exported_signal_attribute_string(self, task, attribute, value): lib_importer.task_handle, ctypes.c_int32] error_code = cfunc( - task, attribute, value.encode('ascii')) + task, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_exported_signal_attribute_uint32(self, task, attribute, value): @@ -5235,7 +5235,7 @@ def set_read_attribute_string(self, task, attribute, value): lib_importer.task_handle, ctypes.c_int32] error_code = cfunc( - task, attribute, value.encode('ascii')) + task, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_read_attribute_uint32(self, task, attribute, value): @@ -5308,7 +5308,7 @@ def set_scale_attribute_string(self, scale_name, attribute, value): ctypes_byte_str, ctypes.c_int32] error_code = cfunc( - scale_name, attribute, value.encode('ascii')) + scale_name, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_timing_attribute_bool(self, task, attribute, value): @@ -5388,7 +5388,7 @@ def set_timing_attribute_ex_string( ctypes.c_int32] error_code = cfunc( - task, device_names, attribute, value.encode('ascii')) + task, device_names, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_timing_attribute_ex_uint32( @@ -5440,7 +5440,7 @@ def set_timing_attribute_string(self, task, attribute, value): lib_importer.task_handle, ctypes.c_int32] error_code = cfunc( - task, attribute, value.encode('ascii')) + task, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_timing_attribute_uint32(self, task, attribute, value): @@ -5536,7 +5536,7 @@ def set_trig_attribute_string(self, task, attribute, value): lib_importer.task_handle, ctypes.c_int32] error_code = cfunc( - task, attribute, value.encode('ascii')) + task, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_trig_attribute_timestamp(self, task, attribute, value): @@ -5612,7 +5612,7 @@ def set_watchdog_attribute_string(self, task, lines, attribute, value): ctypes.c_int32] error_code = cfunc( - task, lines, attribute, value.encode('ascii')) + task, lines, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_write_attribute_bool(self, task, attribute, value): @@ -5660,7 +5660,7 @@ def set_write_attribute_string(self, task, attribute, value): lib_importer.task_handle, ctypes.c_int32] error_code = cfunc( - task, attribute, value.encode('ascii')) + task, attribute, value.encode(lib_importer.encoding)) self.check_for_error(error_code) def set_write_attribute_uint32(self, task, attribute, value): @@ -6225,7 +6225,7 @@ def get_error_string(self, error_code): if query_error_code < 0: _logger.error('Failed to get error string for error code %d. DAQmxGetErrorString returned error code %d.', error_code, query_error_code) return 'Failed to retrieve error description.' - return error_buffer.value.decode('utf-8') + return error_buffer.value.decode(lib_importer.encoding) def get_extended_error_info(self): error_buffer = ctypes.create_string_buffer(2048) @@ -6240,7 +6240,7 @@ def get_extended_error_info(self): if query_error_code < 0: _logger.error('Failed to get extended error info. DAQmxGetExtendedErrorInfo returned error code %d.', query_error_code) return 'Failed to retrieve error description.' - return error_buffer.value.decode('utf-8') + return error_buffer.value.decode(lib_importer.encoding) def read_power_binary_i16( self, task, num_samps_per_chan, timeout, fill_mode, diff --git a/src/codegen/templates/_library_interpreter.py.mako b/src/codegen/templates/_library_interpreter.py.mako index f5346554..3ff0a96d 100644 --- a/src/codegen/templates/_library_interpreter.py.mako +++ b/src/codegen/templates/_library_interpreter.py.mako @@ -117,7 +117,7 @@ class LibraryInterpreter(BaseInterpreter): if query_error_code < 0: _logger.error('Failed to get error string for error code %d. DAQmxGetErrorString returned error code %d.', error_code, query_error_code) return 'Failed to retrieve error description.' - return error_buffer.value.decode('utf-8') + return error_buffer.value.decode(lib_importer.encoding) ## get_extended_error_info has special error handling and it is library-only because it uses ## thread-local storage. @@ -134,7 +134,7 @@ class LibraryInterpreter(BaseInterpreter): if query_error_code < 0: _logger.error('Failed to get extended error info. DAQmxGetExtendedErrorInfo returned error code %d.', query_error_code) return 'Failed to retrieve error description.' - return error_buffer.value.decode('utf-8') + return error_buffer.value.decode(lib_importer.encoding) ## The metadata for 'read_power_binary_i16' function is not available in daqmxAPISharp.json file. def read_power_binary_i16( diff --git a/src/codegen/utilities/interpreter_helpers.py b/src/codegen/utilities/interpreter_helpers.py index 49c6566c..02dfdc37 100644 --- a/src/codegen/utilities/interpreter_helpers.py +++ b/src/codegen/utilities/interpreter_helpers.py @@ -430,7 +430,7 @@ def get_return_values(func): f"[{param.parameter_name}_element.value for {param.parameter_name}_element in {param.parameter_name}]" ) elif param.ctypes_data_type == "ctypes.c_char_p": - return_values.append(f"{param.parameter_name}.value.decode('ascii')") + return_values.append(f"{param.parameter_name}.value.decode(lib_importer.encoding)") elif param.is_list: if is_read_write_function: return_values.append(param.parameter_name) @@ -658,7 +658,7 @@ def get_read_array_parameters(func): def type_cast_attribute_set_function_parameter(param): """Type casting of attribute set parameter during c call.""" if param.ctypes_data_type == "ctypes.c_char_p": - return f"{param.parameter_name}.encode('ascii')" + return f"{param.parameter_name}.encode(lib_importer.encoding)" if is_numpy_array_datatype(param): return f"{param.parameter_name}.ctypes.data_as(ctypes.c_void_p)" return f"{param.ctypes_data_type}({param.parameter_name})" diff --git a/src/handwritten/_lib.py b/src/handwritten/_lib.py index 33b9e21b..6789d402 100644 --- a/src/handwritten/_lib.py +++ b/src/handwritten/_lib.py @@ -6,6 +6,8 @@ import platform import sys import threading +import locale +from decouple import config from typing import cast, TYPE_CHECKING from nidaqmx.errors import DaqNotFoundError, DaqNotSupportedError, DaqFunctionNotSupportedError @@ -46,13 +48,13 @@ def _setter(self, val): class CtypesByteString: """ - Custom argtype that automatically converts unicode strings to ASCII - strings in Python 3. + Custom argtype that automatically converts unicode strings to encoding + used by the DAQmx C API DLL in Python 3. """ @classmethod def from_param(cls, param): if isinstance(param, str): - param = param.encode('ascii') + param = param.encode(lib_importer.encoding) return ctypes.c_char_p(param) @@ -128,6 +130,7 @@ def __init__(self): self._cdll = None self._cal_handle = None self._task_handle = None + self._encoding = None @property def windll(self): @@ -148,32 +151,65 @@ def task_handle(self) -> type: @property def cal_handle(self) -> type: return CalHandle - + + @property + def encoding(self): + if self._encoding is None: + self._import_lib() + return self._encoding + def _import_lib(self): """ Determines the location of and loads the NI-DAQmx CAI DLL. """ self._windll = None self._cdll = None + self._encoding = None windll = None cdll = None - - if sys.platform.startswith('win') or sys.platform.startswith('cli'): - try: - if 'iron' in platform.python_implementation().lower(): - windll = ctypes.windll.nicaiu - cdll = ctypes.cdll.nicaiu - else: - windll = ctypes.windll.LoadLibrary('nicaiu') - cdll = ctypes.cdll.LoadLibrary('nicaiu') - except (OSError, WindowsError) as e: - raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) from e + encoding = None + + if sys.platform.startswith('win'): + + def _load_lib(libname: str): + windll = ctypes.windll.LoadLibrary(libname) + cdll = ctypes.cdll.LoadLibrary(libname) + return windll, cdll + + # Feature Toggle to load nicaiu.dll or nicai_utf8.dll + # The Feature Toggle can be set in the .env file + nidaqmx_c_library = config('NIDAQMX_C_LIBRARY', default=None) + + if nidaqmx_c_library is not None: + try: + if nidaqmx_c_library=="nicaiu": + windll, cdll = _load_lib("nicaiu") + encoding = locale.getlocale()[1] + elif nidaqmx_c_library=="nicai_utf8": + windll, cdll = _load_lib("nicai_utf8") + encoding = 'utf-8' + else: + raise ValueError(f"Unsupported NIDAQMX_C_LIBRARY value: {nidaqmx_c_library}") + except (OSError, WindowsError) as e: + raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) from e + else: + try: + windll, cdll = _load_lib("nicai_utf8") + encoding = 'utf-8' + except (OSError, WindowsError): + # Fallback to nicaiu.dll if nicai_utf8.dll cannot be loaded + try: + windll, cdll = _load_lib("nicaiu") + encoding = locale.getlocale()[1] + except (OSError, WindowsError) as e: + raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) from e elif sys.platform.startswith('linux'): # On linux you can use the command find_library('nidaqmx') if find_library('nidaqmx') is not None: cdll = ctypes.cdll.LoadLibrary(find_library('nidaqmx')) windll = cdll + encoding = locale.getlocale()[1] else: raise DaqNotFoundError(_DAQ_NOT_FOUND_MESSAGE) else: @@ -181,6 +217,7 @@ def _import_lib(self): self._windll = DaqFunctionImporter(windll) self._cdll = DaqFunctionImporter(cdll) + self._encoding = encoding lib_importer = DaqLibImporter()