From b0de03f1fdb7ba0824b67c53bfa90d24d90002e4 Mon Sep 17 00:00:00 2001 From: Henry Pinkard Date: Tue, 27 Oct 2020 21:53:49 -0400 Subject: [PATCH 1/2] change encodings to fix SLM bugs --- .../micromanager/internal/zmq/ZMQServer.java | 17 ++++----- .../micromanager/internal/zmq/ZMQUtil.java | 10 +++--- pycromanager/core.py | 35 ++++++++++--------- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java b/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java index 9887ccd3..5071c21d 100644 --- a/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java +++ b/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java @@ -11,6 +11,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -110,7 +111,7 @@ public void initialize(int port) { String exceptionAsString = sw.toString(); json.put("value", exceptionAsString); - reply = json.toString().getBytes(); + reply = json.toString().getBytes( StandardCharsets.ISO_8859_1); e.printStackTrace(); } catch (JSONException ex) { @@ -141,7 +142,7 @@ protected byte[] getField(Object obj, JSONObject json) throws JSONException, NoS Object field = obj.getClass().getField(fieldName).get(obj); JSONObject serialized = new JSONObject(); util_.serialize(field, serialized, port_); - return serialized.toString().getBytes(); + return serialized.toString().getBytes(StandardCharsets.ISO_8859_1); } protected void setField(Object obj, JSONObject json) throws JSONException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException { @@ -342,7 +343,7 @@ private byte[] runMethod(Object obj, JSONObject message) throws NoSuchMethodExce JSONObject serialized = new JSONObject(); util_.serialize(result, serialized, port_); - return serialized.toString().getBytes(); + return serialized.toString().getBytes( StandardCharsets.ISO_8859_1); } protected byte[] parseAndExecuteCommand(String message) throws Exception { @@ -356,14 +357,14 @@ protected byte[] parseAndExecuteCommand(String message) throws Exception { reply = new JSONObject(); reply.put("type", "none"); reply.put("version", VERSION); - return reply.toString().getBytes(); + return reply.toString().getBytes( StandardCharsets.ISO_8859_1); } case "get-constructors": { String classpath = request.getString("classpath"); reply = new JSONObject(); reply.put("type", "none"); reply.put("api", ZMQUtil.parseConstructors(classpath, classMapper_)); - return reply.toString().getBytes(); + return reply.toString().getBytes( StandardCharsets.ISO_8859_1); } case "constructor": { //construct a new object (or grab an exisitng instance) Class baseClass = util_.loadClass(request.getString("classpath")); @@ -385,7 +386,7 @@ protected byte[] parseAndExecuteCommand(String message) throws Exception { } reply = new JSONObject(); util_.serialize(instance, reply, port_); - return reply.toString().getBytes(); + return reply.toString().getBytes( StandardCharsets.ISO_8859_1); } case "run-method": { String hashCode = request.getString("hash-code"); @@ -403,7 +404,7 @@ protected byte[] parseAndExecuteCommand(String message) throws Exception { setField(target, request); reply = new JSONObject(); reply.put("type", "none"); - return reply.toString().getBytes(); + return reply.toString().getBytes( StandardCharsets.ISO_8859_1); } case "destructor": { String hashCode = request.getString("hash-code"); @@ -413,7 +414,7 @@ protected byte[] parseAndExecuteCommand(String message) throws Exception { reply = new JSONObject(); reply.put("type", "none"); - return reply.toString().getBytes(); + return reply.toString().getBytes( StandardCharsets.ISO_8859_1); } default: break; diff --git a/java/src/main/java/org/micromanager/internal/zmq/ZMQUtil.java b/java/src/main/java/org/micromanager/internal/zmq/ZMQUtil.java index 8a94ac82..8bf2ef4b 100644 --- a/java/src/main/java/org/micromanager/internal/zmq/ZMQUtil.java +++ b/java/src/main/java/org/micromanager/internal/zmq/ZMQUtil.java @@ -11,8 +11,8 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ShortBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Base64; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; @@ -314,11 +314,12 @@ public static String encodeArray(Object array) { buffer.order(BYTE_ORDER).asFloatBuffer().put((float[]) array); byteArray = buffer.array(); } - return Base64.getEncoder().encodeToString(byteArray); + return new String(byteArray, StandardCharsets.ISO_8859_1); } public static Object decodeArray(String serialized, Class arrayClass) { - byte[] byteArray = Base64.getDecoder().decode(serialized); + byte[] byteArray; + byteArray = serialized.getBytes(StandardCharsets.ISO_8859_1); if (arrayClass.equals(byte[].class)) { return byteArray; } else if (arrayClass.equals(short[].class)) { @@ -623,7 +624,8 @@ static Object convertToPrimitiveClass(Object primitive, Class argClass) { } public static Object convertToPrimitiveArray(Object argClass, String value) { - byte[] bytes = value.getBytes(); + byte[] bytes = new byte[0]; + bytes = value.getBytes( StandardCharsets.ISO_8859_1); if (argClass.equals(byte[].class)) { return bytes; } else if (argClass.equals(short[].class)) { diff --git a/pycromanager/core.py b/pycromanager/core.py index 3b3e28ab..b1586822 100644 --- a/pycromanager/core.py +++ b/pycromanager/core.py @@ -3,7 +3,6 @@ import time import typing import warnings -from base64 import standard_b64encode, standard_b64decode import inspect import numpy as np import zmq @@ -46,8 +45,8 @@ def __del__(self): def _convert_np_to_python(self, d): """ - recursive ply search dictionary and convert any values from numpy floats/ints to - python floats/ints so they can be hson serialized + recursively search dictionary and convert any values from numpy floats/ints to + python floats/ints so they can be json serialized :return: """ if type(d) != dict: @@ -72,12 +71,12 @@ def send(self, message, timeout=0): if self._debug: print('DEBUG, sending: {}'.format(message)) if timeout == 0: - self._socket.send(bytes(message_string, 'utf-8')) + self._socket.send(bytes(message_string, 'iso-8859-1')) else: start = time.time() while 1000 * (time.time() - start) < timeout: try: - self._socket.send(bytes(message_string, 'utf-8'), flags=zmq.NOBLOCK) + self._socket.send(bytes(message_string, 'iso-8859-1'), flags=zmq.NOBLOCK) return True except zmq.ZMQError: pass #ignore, keep trying @@ -98,7 +97,7 @@ def receive(self, timeout=0): pass #ignore, keep trying if reply is None: return reply - message = json.loads(reply.decode('utf-8')) + message = json.loads(reply.decode('iso-8859-1')) if self._debug: print('DEBUG, recieved: {}'.format(message)) self._check_exception(message) @@ -420,7 +419,7 @@ def _deserialize(self, json_return): def serialize_array(array): - return standard_b64encode(array.tobytes()).decode('utf-8') + return array.tobytes().decode('iso-8859-1') def deserialize_array(json_return): @@ -430,16 +429,18 @@ def deserialize_array(json_return): ---------- json_return """ - if json_return['type'] == 'byte-array': - return np.frombuffer(standard_b64decode(json_return['value']), dtype='>u1').copy() - elif json_return['type'] == 'double-array': - return np.frombuffer(standard_b64decode(json_return['value']), dtype='>f8').copy() - elif json_return['type'] == 'int-array': - return np.frombuffer(standard_b64decode(json_return['value']), dtype='>u4').copy() - elif json_return['type'] == 'short-array': - return np.frombuffer(standard_b64decode(json_return['value']), dtype='>u2').copy() - elif json_return['type'] == 'float-array': - return np.frombuffer(standard_b64decode(json_return['value']), dtype='>f4').copy() + if json_return['type'] in ['byte-array', 'int-array', 'short-array', 'float-array']: + decoded = bytes(json_return['value'], 'iso-8859-1') + if json_return['type'] == 'byte-array': + return np.frombuffer(decoded, dtype='>u1').copy() + elif json_return['type'] == 'double-array': + return np.frombuffer(decoded, dtype='>f8').copy() + elif json_return['type'] == 'int-array': + return np.frombuffer(decoded, dtype='>u4').copy() + elif json_return['type'] == 'short-array': + return np.frombuffer(decoded, dtype='>u2').copy() + elif json_return['type'] == 'float-array': + return np.frombuffer(decoded, dtype='>f4').copy() def _package_arguments(valid_method_spec, fn_args): From db2eec3440d8eb2fd9e5aaade32e51696bb88f7b Mon Sep 17 00:00:00 2001 From: Henry Pinkard Date: Tue, 27 Oct 2020 21:57:46 -0400 Subject: [PATCH 2/2] bump versions --- java/pom.xml | 2 +- java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java | 2 +- pycromanager/_version.py | 2 +- pycromanager/core.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index c9a3eb1a..6050267a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.micro-manager.pycro-manager PycroManagerJava - 0.9.5 + 0.10.0 jar 1.8 diff --git a/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java b/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java index 5071c21d..874287f7 100644 --- a/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java +++ b/java/src/main/java/org/micromanager/internal/zmq/ZMQServer.java @@ -34,7 +34,7 @@ public class ZMQServer extends ZMQSocketWrapper { private static Set packages_; private static ZMQUtil util_; - public static final String VERSION = "2.7.0"; + public static final String VERSION = "3.0.0"; private static Function classMapper_; private static ZMQServer masterServer_; diff --git a/pycromanager/_version.py b/pycromanager/_version.py index 08038632..ca8f4cab 100644 --- a/pycromanager/_version.py +++ b/pycromanager/_version.py @@ -1,2 +1,2 @@ -version_info = (0, 5, 1) +version_info = (0, 6, 0) __version__ = ".".join(map(str, version_info)) diff --git a/pycromanager/core.py b/pycromanager/core.py index 4a897312..4f822497 100644 --- a/pycromanager/core.py +++ b/pycromanager/core.py @@ -118,7 +118,7 @@ class Bridge: """ _DEFAULT_PORT = 4827 - _EXPECTED_ZMQ_SERVER_VERSION = "2.7.0" + _EXPECTED_ZMQ_SERVER_VERSION = "3.0.0" thread_local = threading.local()