diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..f41fe296 --- /dev/null +++ b/.clang-format @@ -0,0 +1,221 @@ +# .clang-format for Qt Creator +# +# This is for clang-format >= 5.0. +# +# The configuration below follows the Qt Creator Coding Rules [1] as closely as +# possible. For documentation of the options, see [2]. +# +# Use ../../tests/manual/clang-format-for-qtc/test.cpp for documenting problems +# or testing changes. +# +# In case you update this configuration please also update the qtcStyle() in src\plugins\clangformat\clangformatutils.cpp +# +# [1] https://doc-snapshots.qt.io/qtcreator-extending/coding-style.html +# [2] https://clang.llvm.org/docs/ClangFormatStyleOptions.html +# +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: DontAlign +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: DontAlign +AlignOperands: true +AlignTrailingComments: false +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortLambdasOnASingleLine: None +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: All +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeComma +BreakAfterJavaFieldAnnotations: true +BreakStringLiterals: true +ColumnLimit: 0 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 8 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - forever # avoids { wrapped to next line + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeCategories: + - Regex: '^ + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..e69de29b diff --git a/assets/OpenCMW_logo.odg b/assets/OpenCMW_logo.odg new file mode 100644 index 00000000..566d3706 Binary files /dev/null and b/assets/OpenCMW_logo.odg differ diff --git a/client/pom.xml b/client/pom.xml new file mode 100644 index 00000000..ae3a1d3b --- /dev/null +++ b/client/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + + io.opencmw + opencmw + ${revision}${sha1}${changelist} + ../pom.xml + + + client + + + OpenCmw, CmwLight, and RESTful client implementations. + + + + + io.opencmw + core + ${revision}${sha1}${changelist} + + + de.gsi.dataset + chartfx-dataset + ${version.chartfx} + + + + com.squareup.okhttp3 + okhttp + ${version.okHttp3} + + + com.squareup.okhttp3 + okhttp-sse + ${version.okHttp3} + + + com.squareup.okhttp3 + mockwebserver + ${version.okHttp3} + test + + + + + diff --git a/client/src/main/java/io/opencmw/client/DataSource.java b/client/src/main/java/io/opencmw/client/DataSource.java new file mode 100644 index 00000000..822ea7b7 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/DataSource.java @@ -0,0 +1,118 @@ +package io.opencmw.client; + +import java.time.Duration; +import java.util.List; + +import org.zeromq.ZContext; +import org.zeromq.ZMQ.Socket; +import org.zeromq.ZMsg; + +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.utils.NoDuplicatesList; + +/** + * Interface for DataSources to be added to an EventStore by a single event loop. + * Should provide a static boolean matches(String address) function to determine whether + * it is eligible for a given address. + */ +public abstract class DataSource { + private static final List IMPLEMENTATIONS = new NoDuplicatesList<>(); + + private DataSource() { + // prevent implementers from implementing default constructor + } + + /** + * Constructor + * @param endpoint Endpoint to subscribe to + */ + public DataSource(final String endpoint) { + if (endpoint == null || endpoint.isBlank() || !getFactory().matches(endpoint)) { + throw new UnsupportedOperationException(this.getClass().getName() + " DataSource Implementation does not support endpoint: " + endpoint); + } + } + + /** + * Factory method to get a DataSource for a given endpoint + * @param endpoint endpoint address + * @return if there is a DataSource implementation for the protocol of the endpoint return a Factory to create a new + * Instance of this DataSource + * @throws UnsupportedOperationException in case there is no valid implementation + */ + public static Factory getFactory(final String endpoint) { + for (Factory factory : IMPLEMENTATIONS) { + if (factory.matches(endpoint)) { + return factory; + } + } + throw new UnsupportedOperationException("No DataSource implementation available for endpoint: " + endpoint); + } + + public static void register(final Factory factory) { + IMPLEMENTATIONS.add(0, factory); // custom added implementations are added in front to be discovered first + } + + /** + * Get Socket to wait for in the event loop. + * The main event thread will wait for data becoming available on this socket. + * The socket might be used to receive the actual data or it might just be used to notify the main thread. + * @return a Socket for the event loop to wait upon + */ + public abstract Socket getSocket(); + + protected abstract Factory getFactory(); + + /** + * Gets called whenever data is available on the DataSoure's socket. + * Should then try to receive data and return any results back to the calling event loop. + * @return null if there is no more data available, a Zero length Zmsg if there was data which was only used internally + * or a ZMsg with [reqId, endpoint, byte[] data, [byte[] optional RBAC token]] + */ + public abstract ZMsg getMessage(); + + /** + * Perform housekeeping tasks like connection management, heartbeats, subscriptions, etc + * @return next time housekeeping duties should be performed + */ + public abstract long housekeeping(); + + /** + * Subscribe to this endpoint + * @param reqId the id to join the result of this subscribe with + * @param rbacToken byte array containing signed body hash-key and corresponding RBAC role + */ + public abstract void subscribe(final String reqId, final String endpoint, final byte[] rbacToken); + + /** + * Unsubscribe from the endpoint of this DataSource. + */ + public abstract void unsubscribe(final String reqId); + + /** + * Perform a get request on this endpoint. + * @param requestId request id which later allows to match the returned value to this query. + * This is the only mandatory parameter, all the following may be null. + * @param endpoint extend the filters originally supplied to the endpoint e.g. "ctx=selector&channel=chanA" + * @param filters The serialised filters which will determine which data to update + * @param data The serialised data which can be used by the get call + * @param rbacToken byte array containing signed body hash-key and corresponding RBAC role + */ + public abstract void get(final String requestId, final String endpoint, final byte[] filters, final byte[] data, final byte[] rbacToken); + + /** + * Perform a set request on this endpoint using additional filters + * @param requestId request id which later allows to match the returned value to this query. + * This is the only mandatory parameter, all the following may be null. + * @param endpoint extend the filters originally supplied to the endpoint e.g. "ctx=selector&channel=chanA" + * @param filters The serialised filters which will determine which data to update + * @param data The serialised data which can be used by the get call + * @param rbacToken byte array containing signed body hash-key and corresponding RBAC role + */ + public abstract void set(final String requestId, final String endpoint, final byte[] filters, final byte[] data, final byte[] rbacToken); + + protected interface Factory { + boolean matches(final String endpoint); + Class getMatchingSerialiserType(final String endpoint); + DataSource newInstance(final ZContext context, final String endpoint, final Duration timeout, final String clientId); + } +} diff --git a/client/src/main/java/io/opencmw/client/DataSourceFilter.java b/client/src/main/java/io/opencmw/client/DataSourceFilter.java new file mode 100644 index 00000000..7ba7d582 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/DataSourceFilter.java @@ -0,0 +1,63 @@ +package io.opencmw.client; + +import io.opencmw.Filter; +import io.opencmw.serialiser.IoSerialiser; + +public class DataSourceFilter implements Filter { + public ReplyType eventType = ReplyType.UNKNOWN; + public Class protocolType; + public String device; + public String property; + public DataSourcePublisher.ThePromisedFuture future; + public String context; + + @Override + public void clear() { + eventType = ReplyType.UNKNOWN; + device = "UNKNOWN"; + property = "UNKNOWN"; + future = null; // NOPMD - have to clear the future because the events are reused + context = ""; + } + + @Override + public void copyTo(final Filter other) { + if (other instanceof DataSourceFilter) { + final DataSourceFilter otherDSF = (DataSourceFilter) other; + otherDSF.eventType = eventType; + otherDSF.device = device; + otherDSF.property = property; + otherDSF.future = future; + otherDSF.context = context; + } + } + + /** + * internal enum to track different get/set/subscribe/... transactions + */ + public enum ReplyType { + SUBSCRIBE(0), + GET(1), + SET(2), + UNSUBSCRIBE(3), + UNKNOWN(-1); + + private final byte id; + ReplyType(int id) { + this.id = (byte) id; + } + + public byte getID() { + return id; + } + + public static ReplyType valueOf(final int id) { + for (ReplyType mode : ReplyType.values()) { + if (mode.getID() == id) { + return mode; + } + } + return UNKNOWN; + } + } +} diff --git a/client/src/main/java/io/opencmw/client/DataSourcePublisher.java b/client/src/main/java/io/opencmw/client/DataSourcePublisher.java new file mode 100644 index 00000000..55d14d37 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/DataSourcePublisher.java @@ -0,0 +1,463 @@ +package io.opencmw.client; + +import static java.util.Objects.requireNonNull; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.EventStore; +import io.opencmw.RingBufferEvent; +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.rbac.RbacProvider; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.utils.CustomFuture; +import io.opencmw.utils.SharedPointer; + +import com.lmax.disruptor.EventHandler; + +/** + * Publishes events from different sources into a common {@link EventStore} and takes care of setting the appropriate + * filters and deserialisation of the domain objects. + * + * The subscribe/unsubscribe/set/get methods can be called from any thread and are decoupled from the actual + * + * @author Alexander Krimmm + * @author rstein + */ +public class DataSourcePublisher implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourcePublisher.class); + private static final AtomicInteger INSTANCE_COUNT = new AtomicInteger(); + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private static final ZFrame EMPTY_FRAME = new ZFrame(EMPTY_BYTE_ARRAY); + public static final int MIN_FRAMES_INTERNAL_MSG = 3; + private final String inprocCtrl = "inproc://dsPublisher#" + INSTANCE_COUNT.incrementAndGet(); + protected final Map> requestFutureMap = new ConcurrentHashMap<>(); // + protected final Map clientMap = new ConcurrentHashMap<>(); // + + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicInteger internalReqIdGenerator = new AtomicInteger(0); + private final EventStore eventStore; + private final ZMQ.Poller poller; + private final ZContext context = new ZContext(1); + private final ZMQ.Socket controlSocket; + private final IoBuffer byteBuffer = new FastByteBuffer(2000); // zero length buffer to use when there is no data to deserialise + private final IoClassSerialiser ioClassSerialiser = new IoClassSerialiser(byteBuffer); + + private final ThreadLocal perThreadControlSocket = new ThreadLocal<>() { // creates a client control socket for each calling thread + private ZMQ.Socket result; + + @Override + public void remove() { + if (result != null) { + result.disconnect(inprocCtrl); + } + super.remove(); + } + + @Override + protected ZMQ.Socket initialValue() { + result = context.createSocket(SocketType.DEALER); + result.connect(inprocCtrl); + return result; + } + }; + private final String clientId; + private final RbacProvider rbacProvider; + + public DataSourcePublisher(final RbacProvider rbacProvider, final EventStore publicationTarget, final String... clientId) { + this(rbacProvider, clientId); + eventStore.register((event, sequence, endOfBatch) -> { + final DataSourceFilter dataSourceFilter = event.getFilter(DataSourceFilter.class); + final ThePromisedFuture future = dataSourceFilter.future; + if (future.replyType == DataSourceFilter.ReplyType.SUBSCRIBE) { + final Class domainClass = future.getRequestedDomainObjType(); + final ZMsg cmwMsg = event.payload.get(ZMsg.class); + requireNonNull(cmwMsg.poll()).getString(Charset.defaultCharset()); // ignore header + final byte[] body = requireNonNull(cmwMsg.poll()).getData(); + final String exc = requireNonNull(cmwMsg.poll()).getString(Charset.defaultCharset()); + Object domainObj = null; + if (body != null && body.length != 0) { + ioClassSerialiser.setDataBuffer(FastByteBuffer.wrap(body)); + domainObj = ioClassSerialiser.deserialiseObject(domainClass); + ioClassSerialiser.setDataBuffer(byteBuffer); // allow received byte array to be released + } + publicationTarget.getRingBuffer().publishEvent((publishEvent, seq, obj) -> { + final TimingCtx contextFilter = publishEvent.getFilter(TimingCtx.class); + final EvtTypeFilter evtTypeFilter = publishEvent.getFilter(EvtTypeFilter.class); + publishEvent.arrivalTimeStamp = event.arrivalTimeStamp; + publishEvent.payload = new SharedPointer<>(); + publishEvent.payload.set(obj); + if (exc != null && !exc.isBlank()) { + publishEvent.throwables.add(new Exception(exc)); + } + try { + contextFilter.setSelector(dataSourceFilter.context, 0); + } catch (IllegalArgumentException e) { + LOGGER.atError().setCause(e).addArgument(dataSourceFilter.context).log("No valid context: {}"); + } + // contextFilter.acqts = msg.dataContext.acqStamp; // needs to be added? + // contextFilter.ctxName = // what should go here? + evtTypeFilter.evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evtTypeFilter.typeName = dataSourceFilter.device + '/' + dataSourceFilter.property; + evtTypeFilter.updateType = EvtTypeFilter.UpdateType.COMPLETE; + }, domainObj); + } else if (future.replyType == DataSourceFilter.ReplyType.GET) { + // get data from socket + final ZMsg cmwMsg = event.payload.get(ZMsg.class); + requireNonNull(cmwMsg.poll()).getString(StandardCharsets.UTF_8); + final byte[] body = requireNonNull(cmwMsg.poll()).getData(); + final String exc = requireNonNull(cmwMsg.poll()).getString(Charset.defaultCharset()); + // deserialise + Object obj = null; + if (body != null && body.length != 0) { + ioClassSerialiser.setDataBuffer(FastByteBuffer.wrap(body)); + obj = ioClassSerialiser.deserialiseObject(future.getRequestedDomainObjType()); + ioClassSerialiser.setDataBuffer(byteBuffer); // allow received byte array to be released + } + // notify future + if (exc == null || exc.isBlank()) { + future.castAndSetReply(obj); + } else { + future.setException(new Exception(exc)); + } + // publish to ring buffer + publicationTarget.getRingBuffer().publishEvent((publishEvent, seq, o) -> { + final TimingCtx contextFilter = publishEvent.getFilter(TimingCtx.class); + final EvtTypeFilter evtTypeFilter = publishEvent.getFilter(EvtTypeFilter.class); + publishEvent.arrivalTimeStamp = event.arrivalTimeStamp; + publishEvent.payload = new SharedPointer<>(); + publishEvent.payload.set(o); + if (exc != null && !exc.isBlank()) { + publishEvent.throwables.add(new Exception(exc)); + } + try { + contextFilter.setSelector(dataSourceFilter.context, 0); + } catch (IllegalArgumentException e) { + LOGGER.atError().setCause(e).addArgument(dataSourceFilter.context).log("No valid context: {}"); + } + // contextFilter.acqts = msg.dataContext.acqStamp; // needs to be added? + // contextFilter.ctxName = // what should go here? + evtTypeFilter.evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evtTypeFilter.typeName = dataSourceFilter.device + '/' + dataSourceFilter.property; + evtTypeFilter.updateType = EvtTypeFilter.UpdateType.COMPLETE; + }, obj); + } else { + // ignore other reply types for now + // todo: publish statistics, connection state and getRequests + LOGGER.atInfo().addArgument(event.payload.get()).log("{}"); + } + }); + } + + public DataSourcePublisher(final RbacProvider rbacProvider, final EventHandler eventHandler, final String... clientId) { + this(rbacProvider, clientId); + eventStore.register(eventHandler); + } + + public DataSourcePublisher(final RbacProvider rbacProvider, final String... clientId) { + poller = context.createPoller(1); + // control socket for adding subscriptions / triggering requests from other threads + controlSocket = context.createSocket(SocketType.DEALER); + controlSocket.bind(inprocCtrl); + poller.register(controlSocket, ZMQ.Poller.POLLIN); + // instantiate event store + eventStore = EventStore.getFactory().setSingleProducer(true).setFilterConfig(DataSourceFilter.class).build(); + // register default handlers // TODO: find out how to do this without having to reference them directly + // DataSource.register(CmwLightClient.FACTORY); + // DataSource.register(RestDataSource.FACTORY); + this.clientId = clientId.length == 1 ? clientId[0] : DataSourcePublisher.class.getName(); + this.rbacProvider = rbacProvider; + } + + public ZContext getContext() { + return context; + } + + public EventStore getEventStore() { + return eventStore; + } + + public Future set(String endpoint, final Class requestedDomainObjType, final Object requestBody, final RbacProvider... rbacProvider) { + return set(endpoint, null, requestBody, requestedDomainObjType, rbacProvider); + } + + /** + * Perform an asynchronous set request on the given device/property. + * Checks if a client for this service already exists and if it does performs the asynchronous get on it, otherwise + * it starts a new client and performs it there. + * + * @param endpoint endpoint address for the property e.g. 'rda3://hostname:port/property?selector&filter', + * file:///path/to/directory, mdp://host:port + * @param requestFilter optional map of optional filters e.g. Map.of("channelName", "VoltageChannel") + * @param requestBody optional domain object payload to be send with the request + * @param requestedDomainObjType the requested result domain object type + * @param The type of the deserialised requested result domain object + * @return A future which will be able to retrieve the deserialised result + */ + public Future set(String endpoint, final Map requestFilter, final Object requestBody, final Class requestedDomainObjType, final RbacProvider... rbacProvider) { + return request(DataSourceFilter.ReplyType.SET, endpoint, requestFilter, requestBody, requestedDomainObjType, rbacProvider); + } + + public Future get(String endpoint, final Class requestedDomainObjType, final RbacProvider... rbacProvider) { + return get(endpoint, null, null, requestedDomainObjType, rbacProvider); + } + + /** + * Perform an asynchronous get request on the given device/property. + * Checks if a client for this service already exists and if it does performs the asynchronous get on it, otherwise + * it starts a new client and performs it there. + * + * @param endpoint endpoint address for the property e.g. 'rda3://hostname:port/property?selector&filter', + * file:///path/to/directory, mdp://host:port + * @param requestFilter optional map of optional filters e.g. Map.of("channelName", "VoltageChannel") + * @param requestBody optional domain object payload to be send with the request + * @param requestedDomainObjType the requested result domain object type + * @param The type of the deserialised requested result domain object + * @return A future which will be able to retrieve the deserialised result + */ + public Future get(String endpoint, final Map requestFilter, final Object requestBody, final Class requestedDomainObjType, final RbacProvider... rbacProvider) { + return request(DataSourceFilter.ReplyType.GET, endpoint, requestFilter, requestBody, requestedDomainObjType, rbacProvider); + } + + private ThePromisedFuture request(final DataSourceFilter.ReplyType replyType, final String endpoint, final Map requestFilter, final Object requestBody, final Class requestedDomainObjType, final RbacProvider... rbacProvider) { + final String requestId = clientId + internalReqIdGenerator.incrementAndGet(); + final ThePromisedFuture requestFuture = newFuture(endpoint, requestFilter, requestBody, requestedDomainObjType, replyType, requestId); + final Class matchingSerialiser = DataSource.getFactory(endpoint).getMatchingSerialiserType(endpoint); + + // signal socket for get with endpoint and request id + final ZMsg msg = new ZMsg(); + msg.add(new byte[] { replyType.getID() }); + msg.add(requestId); + msg.add(endpoint); + if (requestFilter == null) { + msg.add(EMPTY_FRAME); + } else { + ioClassSerialiser.getDataBuffer().reset(); + ioClassSerialiser.setMatchedIoSerialiser(matchingSerialiser); // needs to be converted in DataSource impl + ioClassSerialiser.serialiseObject(requestFilter); + msg.add(Arrays.copyOfRange(ioClassSerialiser.getDataBuffer().elements(), 0, ioClassSerialiser.getDataBuffer().position())); + } + if (requestBody == null) { + msg.add(EMPTY_FRAME); + } else { + ioClassSerialiser.getDataBuffer().reset(); + ioClassSerialiser.setMatchedIoSerialiser(matchingSerialiser); // needs to be converted in DataSource impl + ioClassSerialiser.serialiseObject(requestBody); + msg.add(Arrays.copyOfRange(ioClassSerialiser.getDataBuffer().elements(), 0, ioClassSerialiser.getDataBuffer().position())); + } + // RBAC + if (rbacProvider.length > 0 || this.rbacProvider != null) { + final RbacProvider rbac = rbacProvider.length > 0 ? rbacProvider[0] : this.rbacProvider; // NOPMD - future use + // rbac.sign(msg); // todo: sign message and add rbac token and signature + } else { + msg.add(EMPTY_FRAME); + } + + msg.send(perThreadControlSocket.get()); + //TODO: do we need the following 'remove()' + perThreadControlSocket.remove(); + return requestFuture; + } + + public void subscribe(final String endpoint, final Class requestedDomainObjType) { + subscribe(endpoint, requestedDomainObjType, null, null); + } + + public String subscribe(final String endpoint, final Class requestedDomainObjType, final Map requestFilter, final Object requestBody, final RbacProvider... rbacProvider) { + ThePromisedFuture future = request(DataSourceFilter.ReplyType.SUBSCRIBE, endpoint, requestFilter, requestBody, requestedDomainObjType, rbacProvider); + return future.internalRequestID; + } + + public void unsubscribe(String requestId) { + // signal socket for get with endpoint and request id + final ZMsg msg = new ZMsg(); + msg.add(new byte[] { DataSourceFilter.ReplyType.UNSUBSCRIBE.getID() }); + msg.add(requestId); + msg.add(requestFutureMap.get(requestId).endpoint); + msg.send(perThreadControlSocket.get()); + //TODO: do we need the following 'remove()' + perThreadControlSocket.remove(); + } + + @Override + public void run() { + // start the ring buffer and its processors + eventStore.start(); + // event loop polling all data sources and performing regular housekeeping jobs + running.set(true); + long nextHousekeeping = System.currentTimeMillis(); // immediately perform first housekeeping + long tout = 0L; + while (!Thread.interrupted() && running.get() && (tout <= 0 || -1 != poller.poll(tout))) { + // get data from clients + boolean dataAvailable = true; + while (dataAvailable && System.currentTimeMillis() < nextHousekeeping && running.get()) { + dataAvailable = handleDataSourceSockets(); + // check specificaly for control socket + dataAvailable |= handleControlSocket(); + } + + nextHousekeeping = clientMap.values().stream().mapToLong(DataSource::housekeeping).min().orElse(System.currentTimeMillis() + 1000); + tout = nextHousekeeping - System.currentTimeMillis(); + } + LOGGER.atDebug().addArgument(clientMap.values()).log("poller returned negative value - abort run() - clients = {}"); + } + + public void start() { + new Thread(this).start(); // NOPMD - not a webapp + } + + protected boolean handleControlSocket() { + final ZMsg controlMsg = ZMsg.recvMsg(controlSocket, false); + if (controlMsg == null) { + return false; // no more data available on control socket + } + if (controlMsg.size() < MIN_FRAMES_INTERNAL_MSG) { // msgType, requestId and endpoint have to be always present + LOGGER.atDebug().log("ignoring invalid message"); + return true; // ignore invalid partial message + } + final DataSourceFilter.ReplyType msgType = DataSourceFilter.ReplyType.valueOf(controlMsg.pollFirst().getData()[0]); + final String requestId = requireNonNull(controlMsg.pollFirst()).getString(Charset.defaultCharset()); + final String endpoint = requireNonNull(controlMsg.pollFirst()).getString(Charset.defaultCharset()); + final byte[] filters = controlMsg.isEmpty() ? EMPTY_BYTE_ARRAY : controlMsg.pollFirst().getData(); + final byte[] data = controlMsg.isEmpty() ? EMPTY_BYTE_ARRAY : controlMsg.pollFirst().getData(); + final byte[] rbacToken = controlMsg.isEmpty() ? EMPTY_BYTE_ARRAY : controlMsg.pollFirst().getData(); + + final DataSource client = getClient(endpoint); // get client for endpoint + switch (msgType) { + case SUBSCRIBE: // subscribe: 0b, requestId, addr/dev/prop?sel&filters, [filter] + client.subscribe(requestId, endpoint, rbacToken); // issue get request + break; + case GET: // get: 1b, reqId, addr/dev/prop?sel&filters, [filter] + client.get(requestId, endpoint, filters, data, rbacToken); // issue get request + break; + case SET: // set: 2b, reqId, addr/dev/prop?sel&filters, data, add data to blocking queue instead? + client.set(requestId, endpoint, filters, data, rbacToken); + break; + case UNSUBSCRIBE: //unsub: 3b, reqId, endpoint + client.unsubscribe(requestId); + requestFutureMap.remove(requestId); + break; + case UNKNOWN: + default: + throw new UnsupportedOperationException("Illegal operation type"); + } + return true; + } + + protected boolean handleDataSourceSockets() { + boolean dataAvailable = false; + for (DataSource entry : clientMap.values()) { + final ZMsg reply = entry.getMessage(); + if (reply == null) { + continue; // no data received, queue empty + } + dataAvailable = true; + if (reply.isEmpty()) { + continue; // there was data received, but only used for internal state of the client + } + // the received data consists of the following frames: replyType(byte), reqId(string), endpoint(string), dataBody(byte[]) + eventStore.getRingBuffer().publishEvent((event, sequence) -> { + final String reqId = requireNonNull(reply.pollFirst()).getString(Charset.defaultCharset()); + final ThePromisedFuture returnFuture = requestFutureMap.get(reqId); + if (returnFuture.getReplyType() != DataSourceFilter.ReplyType.SUBSCRIBE) { // remove entries for one time replies + assert returnFuture.getInternalRequestID().equals(reqId) + : "requestID mismatch"; + requestFutureMap.remove(reqId); + } + final Endpoint endpoint = new Endpoint(requireNonNull(reply.pollFirst()).getString(Charset.defaultCharset())); // NOPMD - need to create new Endpoint + event.arrivalTimeStamp = System.currentTimeMillis(); + event.payload = new SharedPointer<>(); // NOPMD - need to create new shared pointer instance + event.payload.set(reply); // ZMsg containing header, body and exception frame + final DataSourceFilter dataSourceFilter = event.getFilter(DataSourceFilter.class); + dataSourceFilter.future = returnFuture; + dataSourceFilter.eventType = DataSourceFilter.ReplyType.SUBSCRIBE; + dataSourceFilter.device = endpoint.getDevice(); + dataSourceFilter.property = endpoint.getProperty(); + dataSourceFilter.context = endpoint.getSelector(); + }); + } + return dataAvailable; + } + + protected ThePromisedFuture newFuture(final String endpoint, final Map requestFilter, final Object requestBody, final Class requestedDomainObjType, final DataSourceFilter.ReplyType replyType, final String requestId) { + final ThePromisedFuture requestFuture = new ThePromisedFuture<>(endpoint, requestFilter, requestBody, requestedDomainObjType, replyType, requestId); + final Object oldEntry = requestFutureMap.put(requestId, requestFuture); + assert oldEntry == null : "requestID '" + requestId + "' already present in requestFutureMap"; + return requestFuture; + } + + private DataSource getClient(final String endpoint) { + return clientMap.computeIfAbsent(new Endpoint(endpoint).getAddress(), requestedEndPoint -> { + final DataSource dataSource = DataSource.getFactory(requestedEndPoint).newInstance(context, endpoint, Duration.ofMillis(100), Long.toString(internalReqIdGenerator.incrementAndGet())); + poller.register(dataSource.getSocket(), ZMQ.Poller.POLLIN); + return dataSource; + }); + } + + public static class ThePromisedFuture extends CustomFuture { // NOPMD - no need for setters/getters here + private final String endpoint; + private final Map requestFilter; + private final Object requestBody; + private final Class requestedDomainObjType; + private final DataSourceFilter.ReplyType replyType; + private final String internalRequestID; + + public ThePromisedFuture(final String endpoint, final Map requestFilter, final Object requestBody, final Class requestedDomainObjType, final DataSourceFilter.ReplyType replyType, final String internalRequestID) { + super(); + this.endpoint = endpoint; + this.requestFilter = requestFilter; + this.requestBody = requestBody; + this.requestedDomainObjType = requestedDomainObjType; + this.replyType = replyType; + this.internalRequestID = internalRequestID; + } + + public String getEndpoint() { + return endpoint; + } + + public DataSourceFilter.ReplyType getReplyType() { + return replyType; + } + + public Object getRequestBody() { + return requestBody; + } + + public Map getRequestFilter() { + return requestFilter; + } + + public Class getRequestedDomainObjType() { + return requestedDomainObjType; + } + + @SuppressWarnings("unchecked") + protected void castAndSetReply(final Object newValue) { + this.setReply((R) newValue); + } + + public String getInternalRequestID() { + return internalRequestID; + } + } +} diff --git a/client/src/main/java/io/opencmw/client/Endpoint.java b/client/src/main/java/io/opencmw/client/Endpoint.java new file mode 100644 index 00000000..74b97db6 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/Endpoint.java @@ -0,0 +1,137 @@ +package io.opencmw.client; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Endpoint helper class to deserialise endpoint strings. + * Uses lazy initialisation to prevent doing unnecessary work or doing the same thing twice. + */ +public class Endpoint { // NOPMD data class + private static final String DEFAULT_SELECTOR = ""; + public static final String FILTER_TYPE_LONG = "long:"; + public static final String FILTER_TYPE_INT = "int:"; + public static final String FILTER_TYPE_BOOL = "bool:"; + public static final String FILTER_TYPE_DOUBLE = "double:"; + public static final String FILTER_TYPE_FLOAT = "float:"; + private final String value; + private String protocol; + private String address; + private String device; + private String property; + private String selector; + private Map filters; + + public Endpoint(final String endpoint) { + this.value = endpoint; + } + + public String getProtocol() { + if (protocol == null) { + parse(); + } + return protocol; + } + + @Override + public String toString() { + return value; + } + + public String getAddress() { + if (protocol == null) { + parse(); + } + return address; + } + + public String getDevice() { + if (protocol == null) { + parse(); + } + return device; + } + + public String getSelector() { + if (protocol == null) { + parse(); + } + return selector; + } + + public String getProperty() { + return property; + } + + public Map getFilters() { + return filters; + } + + public String getEndpointForContext(final String context) { + if (context == null || context.equals("")) { + return value; + } + parse(); + final String filterString = filters.entrySet().stream() // + .map(e -> { + String val; + if (e.getValue() instanceof String) { + val = (String) e.getValue(); + } else if (e.getValue() instanceof Integer) { + val = FILTER_TYPE_INT + e.getValue(); + } else if (e.getValue() instanceof Long) { + val = FILTER_TYPE_LONG + e.getValue(); + } else if (e.getValue() instanceof Boolean) { + val = FILTER_TYPE_BOOL + e.getValue(); + } else if (e.getValue() instanceof Double) { + val = FILTER_TYPE_DOUBLE + e.getValue(); + } else if (e.getValue() instanceof Float) { + val = FILTER_TYPE_FLOAT + e.getValue(); + } else { + throw new UnsupportedOperationException("Data type not supported in endpoint filters"); + } + return e.getKey() + '=' + val; + }) // + .collect(Collectors.joining("&")); + return address + '/' + device + '/' + property + "?ctx=" + context + '&' + filterString; + } + + private void parse() { + final String[] tmp = value.split("\\?", 2); // split into address/dev/prop and sel+filters part + final String[] adp = tmp[0].split("/"); // split access point into parts + device = adp[adp.length - 2]; // get device name from access point + property = adp[adp.length - 1]; // get property name from access point + address = tmp[0].substring(0, tmp[0].length() - device.length() - property.length() - 2); + protocol = address.substring(0, address.indexOf("://") + 3); + filters = new HashMap<>(); + selector = DEFAULT_SELECTOR; + filters = new HashMap<>(); + + final String paramString = tmp[1]; + final String[] kvpairs = paramString.split("&"); // split into individual key/value pairs + for (final String pair : kvpairs) { + String[] splitpair = pair.split("=", 2); // split at first equal sign + if (splitpair.length != 2) { + continue; + } + if ("ctx".equals(splitpair[0])) { + selector = splitpair[1]; + } else { + if (splitpair[1].startsWith(FILTER_TYPE_INT)) { + filters.put(splitpair[0], Integer.valueOf(splitpair[1].substring(FILTER_TYPE_INT.length()))); + } else if (splitpair[1].startsWith(FILTER_TYPE_LONG)) { + filters.put(splitpair[0], Long.valueOf(splitpair[1].substring(FILTER_TYPE_LONG.length()))); + } else if (splitpair[1].startsWith(FILTER_TYPE_BOOL)) { + filters.put(splitpair[0], Boolean.valueOf(splitpair[1].substring(FILTER_TYPE_BOOL.length()))); + } else if (splitpair[1].startsWith(FILTER_TYPE_DOUBLE)) { + filters.put(splitpair[0], Double.valueOf(splitpair[1].substring(FILTER_TYPE_DOUBLE.length()))); + } else if (splitpair[1].startsWith(FILTER_TYPE_FLOAT)) { + filters.put(splitpair[0], Float.valueOf(splitpair[1].substring(FILTER_TYPE_FLOAT.length()))); + } else { + filters.put(splitpair[0], splitpair[1]); + } + } + } + } +} diff --git a/client/src/main/java/io/opencmw/client/cmwlight/CmwLightDataSource.java b/client/src/main/java/io/opencmw/client/cmwlight/CmwLightDataSource.java new file mode 100644 index 00000000..d92ff899 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/cmwlight/CmwLightDataSource.java @@ -0,0 +1,545 @@ +package io.opencmw.client.cmwlight; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMQException; +import org.zeromq.ZMsg; + +import io.opencmw.client.DataSource; +import io.opencmw.client.Endpoint; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.CmwLightSerialiser; + +/** + * A lightweight implementation of the CMW RDA3 client part. + * Reads all sockets from a single Thread, which can also be embedded into other event loops. + * Manages connection state and automatically reconnects broken connections and subscriptions. + */ +public class CmwLightDataSource extends DataSource { // NOPMD - class should probably be smaller + private static final Logger LOGGER = LoggerFactory.getLogger(CmwLightDataSource.class); + private static final AtomicLong CONNECTION_ID_GENERATOR = new AtomicLong(0); // global counter incremented for each connection + private static final AtomicInteger REQUEST_ID_GENERATOR = new AtomicInteger(0); + public static final String RDA_3_PROTOCOL = "rda3://"; + public static final Factory FACTORY = new Factory() { + @Override + public boolean matches(final String endpoint) { + return endpoint.startsWith(RDA_3_PROTOCOL); + } + + @Override + public Class getMatchingSerialiserType(final String endpoint) { + return CmwLightSerialiser.class; + } + + @Override + public DataSource newInstance(final ZContext context, final String endpoint, final Duration timeout, final String clientId) { + return new CmwLightDataSource(context, endpoint, clientId); + } + }; + protected static final int HEARTBEAT_INTERVAL = 1000; // time between to heartbeats in ms + protected static final int HEARTBEAT_ALLOWED_MISSES = 3; // number of heartbeats which can be missed before resetting the conection + protected static final long SUBSCRIPTION_TIMEOUT = 1000; // maximum time after which a connection should be reconnected + private static DirectoryLightClient directoryLightClient; + protected final AtomicInteger channelId = new AtomicInteger(0); // connection local counter incremented for each channel + protected final ZContext context; + protected final ZMQ.Socket socket; + protected final AtomicReference connectionState = new AtomicReference<>(ConnectionState.DISCONNECTED); + private final String address; + protected final String sessionId; + protected long connectionId; + protected final Map subscriptions = new HashMap<>(); // all subscriptions added to the server // NOPMD - only accessed from main thread + protected final Map subscriptionsByReqId = new HashMap<>(); // all subscriptions added to the server // NOPMD - only accessed from main thread + protected final Map replyIdMap = new HashMap<>(); // all acknowledged subscriptions by their reply id // NOPMD - only accessed from main thread + protected long lastHbReceived = -1; + protected long lastHbSent = -1; + protected int backOff = 20; + private final Queue queuedRequests = new LinkedBlockingQueue<>(); + private final Map pendingRequests = new HashMap<>(); // NOPMD - only accessed from main thread + private String connectedAddress = ""; + + public CmwLightDataSource(final ZContext context, final String endpoint, final String clientId) { + super(endpoint); + LOGGER.atTrace().addArgument(endpoint).log("connecting to: {}"); + this.context = context; + this.socket = context.createSocket(SocketType.DEALER); + this.sessionId = getSessionId(clientId); + this.address = new Endpoint(endpoint).getAddress(); + } + + public static DirectoryLightClient getDirectoryLightClient() { + return directoryLightClient; + } + + public static void setDirectoryLightClient(final DirectoryLightClient directoryLightClient) { + CmwLightDataSource.directoryLightClient = directoryLightClient; + } + + public CmwLightMessage receiveData() { + // receive data + try { + final ZMsg data = ZMsg.recvMsg(socket, ZMQ.DONTWAIT); + if (data == null) { + return null; + } + return CmwLightProtocol.parseMsg(data); + } catch (CmwLightProtocol.RdaLightException e) { + LOGGER.atDebug().setCause(e).log("error parsing cmw light reply: "); + return null; + } + } + @Override + public ZMsg getMessage() { // return maintenance objects instead of replies + final long currentTime = System.currentTimeMillis(); // NOPMD + CmwLightMessage reply = receiveData(); + if (reply == null) { + return null; + } + switch (reply.messageType) { + case SERVER_CONNECT_ACK: + if (connectionState.get().equals(ConnectionState.CONNECTING)) { + LOGGER.atTrace().addArgument(connectedAddress).log("Connected to server: {}"); + connectionState.set(ConnectionState.CONNECTED); + lastHbReceived = currentTime; + backOff = 20; // reset back-off time + } else { + LOGGER.atWarn().addArgument(reply).log("ignoring unsolicited connection acknowledgement: {}"); + } + return new ZMsg(); + case SERVER_HB: + if (connectionState.get() != ConnectionState.CONNECTED) { + LOGGER.atWarn().addArgument(reply).log("ignoring heartbeat received before connection established: {}"); + return new ZMsg(); + } + lastHbReceived = currentTime; + return new ZMsg(); + case SERVER_REP: + if (connectionState.get() != ConnectionState.CONNECTED) { + LOGGER.atWarn().addArgument(reply).log("ignoring data received before connection established: {}"); + return new ZMsg(); + } + lastHbReceived = currentTime; + return handleServerReply(reply, currentTime); + case CLIENT_CONNECT: + case CLIENT_REQ: + case CLIENT_HB: + default: + LOGGER.atWarn().addArgument(reply).log("ignoring client message from server: {}"); + return new ZMsg(); + } + } + + private ZMsg handleServerReply(final CmwLightMessage reply, final long currentTime) { //NOPMD + final ZMsg result = new ZMsg(); + switch (reply.requestType) { + case REPLY: + Request requestForReply = pendingRequests.remove(reply.id); + result.add(requestForReply.requestId); + result.add(new Endpoint(requestForReply.endpoint).getEndpointForContext(reply.dataContext.cycleName)); + result.add(new ZFrame(new byte[0])); // header + result.add(reply.bodyData); // body + result.add(new ZFrame(new byte[0])); // exception + return result; + case EXCEPTION: + final Request requestForException = pendingRequests.remove(reply.id); + result.add(requestForException.requestId); + result.add(requestForException.endpoint); + result.add(new ZFrame(new byte[0])); // header + result.add(new ZFrame(new byte[0])); // body + result.add(reply.exceptionMessage.message); // exception + return result; + case SUBSCRIBE: + final long id = reply.id; + final Subscription sub = subscriptions.get(id); + sub.updateId = (long) reply.options.get(CmwLightProtocol.FieldName.SOURCE_ID_TAG.value()); + replyIdMap.put(sub.updateId, sub); + sub.subscriptionState = SubscriptionState.SUBSCRIBED; + LOGGER.atDebug().addArgument(sub.device).addArgument(sub.property).log("subscription successful: {}/{}"); + sub.backOff = 20; + return result; + case UNSUBSCRIBE: + // successfully removed subscription + final Subscription subscriptionForUnsub = subscriptions.remove(reply.id); + subscriptionsByReqId.remove(subscriptionForUnsub.idString); + replyIdMap.remove(subscriptionForUnsub.updateId); + return result; + case NOTIFICATION_DATA: + final Subscription subscriptionForNotification = replyIdMap.get(reply.id); + if (subscriptionForNotification == null) { + LOGGER.atInfo().addArgument(reply.toString()).log("Got unsolicited subscription data: {}"); + return result; + } + result.add(subscriptionForNotification.idString); + result.add(new Endpoint(subscriptionForNotification.endpoint).getEndpointForContext(reply.dataContext.cycleName)); + result.add(new ZFrame(new byte[0])); // header + result.add(reply.bodyData); // body + result.add(new ZFrame(new byte[0])); // exception + return result; + case NOTIFICATION_EXC: + final Subscription subscriptionForNotifyExc = replyIdMap.get(reply.id); + if (subscriptionForNotifyExc == null) { + LOGGER.atInfo().addArgument(reply.toString()).log("Got unsolicited subscription notification error: {}"); + return result; + } + result.add(subscriptionForNotifyExc.idString); + result.add(subscriptionForNotifyExc.endpoint); + result.add(new ZFrame(new byte[0])); // header + result.add(new ZFrame(new byte[0])); // body + result.add(reply.exceptionMessage.message); // exception + return result; + case SUBSCRIBE_EXCEPTION: + final Subscription subForSubExc = subscriptions.get(reply.id); + subForSubExc.subscriptionState = SubscriptionState.UNSUBSCRIBED; + subForSubExc.timeoutValue = currentTime + subForSubExc.backOff; + subForSubExc.backOff *= 2; + LOGGER.atDebug().addArgument(subForSubExc.device).addArgument(subForSubExc.property).log("exception during subscription, retrying: {}/{}"); + result.add(subForSubExc.idString); + result.add(subForSubExc.endpoint); + result.add(new ZFrame(new byte[0])); // header + result.add(new ZFrame(new byte[0])); // body + result.add(reply.exceptionMessage.message); // exception + return result; + // unsupported or non-actionable replies + case GET: + case SET: + case CONNECT: + case EVENT: + case SESSION_CONFIRM: + default: + return result; + } + } + + public enum ConnectionState { + DISCONNECTED, + CONNECTING, + CONNECTED + } + + @Override + public void get(final String requestId, final String endpoint, final byte[] filters, final byte[] data, final byte[] rbacToken) { + final Request request = new Request(CmwLightProtocol.RequestType.GET, requestId, endpoint, filters, data, rbacToken); + queuedRequests.add(request); + } + + @Override + public void set(final String requestId, final String endpoint, final byte[] filters, final byte[] data, final byte[] rbacToken) { + final Request request = new Request(CmwLightProtocol.RequestType.SET, requestId, endpoint, filters, data, rbacToken); + queuedRequests.add(request); + } + + @Override + public void subscribe(final String reqId, final String endpoint, final byte[] rbacToken) { + final Endpoint ep = new Endpoint(endpoint); + final Subscription sub = new Subscription(endpoint, ep.getDevice(), ep.getProperty(), ep.getSelector(), ep.getFilters()); + sub.idString = reqId; + subscriptions.put(sub.id, sub); + subscriptionsByReqId.put(reqId, sub); + } + + @Override + public void unsubscribe(final String reqId) { + subscriptionsByReqId.get(reqId).subscriptionState = SubscriptionState.CANCELED; + } + + public ConnectionState getConnectionState() { + return connectionState.get(); + } + + public ZContext getContext() { + return context; + } + + @Override + public ZMQ.Socket getSocket() { + return socket; + } + + @Override + protected Factory getFactory() { + return FACTORY; + } + + private String getIdentity() { + String hostname; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + hostname = "localhost"; + } + final long processId = ProcessHandle.current().pid(); + connectionId = CONNECTION_ID_GENERATOR.incrementAndGet(); + final int chId = this.channelId.incrementAndGet(); + return hostname + '/' + processId + '/' + connectionId + '/' + chId; + } + + private String getSessionId(final String clientId) { + return "cmwLightClient{pid=" + ProcessHandle.current().pid() + ", conn=" + connectionId + ", clientId=" + clientId + '}'; // Todo: create identification string, cmw uses string with user/app name, pid, etc + } + + public void connect() { + if (connectionState.getAndSet(ConnectionState.CONNECTING) != ConnectionState.DISCONNECTED) { + return; // already connected + } + String address = this.address.startsWith(RDA_3_PROTOCOL) ? this.address.substring(RDA_3_PROTOCOL.length()) : this.address; + if (!address.contains(":")) { + try { + DirectoryLightClient.Device device = directoryLightClient.getDeviceInfo(Collections.singletonList(address)).get(0); + LOGGER.atTrace().addArgument(address).addArgument(device).log("resolved address for device {}: {}"); + address = device.servers.stream().findFirst().orElseThrow().get("Address:"); + } catch (NullPointerException | NoSuchElementException | DirectoryLightClient.DirectoryClientException e) { // NOPMD - directory client must be refactored anyway + LOGGER.atDebug().addArgument(e.getMessage()).log("Error resolving device from nameserver, using address from endpoint. Error was: {}"); + backOff = backOff * 2; + connectionState.set(ConnectionState.DISCONNECTED); + return; + } + } + lastHbSent = System.currentTimeMillis(); + try { + final String identity = getIdentity(); + connectedAddress = "tcp://" + address; + LOGGER.atDebug().addArgument(connectedAddress).addArgument(identity).log("connecting to: {} with identity {}"); + socket.setIdentity(identity.getBytes()); // hostname/process/id/channel + socket.connect(connectedAddress); + CmwLightProtocol.sendMsg(socket, CmwLightMessage.connect(CmwLightProtocol.VERSION)); + } catch (ZMQException | CmwLightProtocol.RdaLightException e) { + LOGGER.atDebug().setCause(e).log("failed to connect: "); + backOff = backOff * 2; + connectionState.set(ConnectionState.DISCONNECTED); + } + } + + private void disconnect() { + LOGGER.atDebug().addArgument(connectedAddress).log("disconnecting {}"); + connectionState.set(ConnectionState.DISCONNECTED); + try { + socket.disconnect(connectedAddress); + } catch (ZMQException e) { + LOGGER.atError().setCause(e).log("Failed to disconnect socket"); + } + // disconnect/reset subscriptions + for (Subscription sub : subscriptions.values()) { + sub.subscriptionState = SubscriptionState.UNSUBSCRIBED; + } + } + + @Override + public long housekeeping() { + final long currentTime = System.currentTimeMillis(); + switch (connectionState.get()) { + case DISCONNECTED: // reconnect after adequate back off + if (currentTime > lastHbSent + backOff) { + LOGGER.atTrace().addArgument(address).log("Connecting to {}"); + connect(); + } + return lastHbSent + backOff; + case CONNECTING: + if (currentTime > lastHbSent + HEARTBEAT_INTERVAL * HEARTBEAT_ALLOWED_MISSES) { // connect timed out -> increase back of and retry + backOff = backOff * 2; + lastHbSent = currentTime; + LOGGER.atTrace().addArgument(connectedAddress).addArgument(backOff).log("Connection timed out for {}, retrying in {} ms"); + disconnect(); + } + return lastHbSent + HEARTBEAT_INTERVAL * HEARTBEAT_ALLOWED_MISSES; + case CONNECTED: + Request request; + while ((request = queuedRequests.poll()) != null) { + pendingRequests.put(request.id, request); + sendRequest(request); + } + if (currentTime > lastHbSent + HEARTBEAT_INTERVAL) { // check for heartbeat interval + // send Heartbeats + sendHeartBeat(); + lastHbSent = currentTime; + // check if heartbeat was received + if (lastHbReceived + HEARTBEAT_INTERVAL * HEARTBEAT_ALLOWED_MISSES < currentTime) { + LOGGER.atDebug().addArgument(backOff).log("Connection timed out, reconnecting in {} ms"); + disconnect(); + return HEARTBEAT_INTERVAL; + } + // check timeouts of connection/subscription requests + for (Subscription sub : subscriptions.values()) { + updateSubscription(currentTime, sub); + } + } + return lastHbSent + HEARTBEAT_INTERVAL; + default: + throw new IllegalStateException("unexpected connection state: " + connectionState.get()); + } + } + + private void sendRequest(final Request request) { + // Filters and data are already serialised but the protocol saves them deserialised :/ + // final ZFrame data = request.data == null ? new ZFrame(new byte[0]) : new ZFrame(request.data); + // final ZFrame filters = request.filters == null ? new ZFrame(new byte[0]) : new ZFrame(request.filters); + final Endpoint requestEndpoint = new Endpoint(request.endpoint); + + try { + switch (request.requestType) { + case GET: + CmwLightProtocol.sendMsg(socket, CmwLightMessage.getRequest( + sessionId, request.id, requestEndpoint.getDevice(), requestEndpoint.getProperty(), + new CmwLightMessage.RequestContext(requestEndpoint.getSelector(), requestEndpoint.getFilters(), null))); + break; + case SET: + Objects.requireNonNull(request.data, "Data for set cannot be null"); + CmwLightProtocol.sendMsg(socket, CmwLightMessage.setRequest( + sessionId, request.id, requestEndpoint.getDevice(), requestEndpoint.getProperty(), + new ZFrame(request.data), + new CmwLightMessage.RequestContext(requestEndpoint.getSelector(), requestEndpoint.getFilters(), null))); + break; + default: + throw new CmwLightProtocol.RdaLightException("Message of unknown type"); + } + } catch (CmwLightProtocol.RdaLightException e) { + LOGGER.atDebug().setCause(e).log("Error sending get request:"); + } + } + + private void updateSubscription(final long currentTime, final Subscription sub) { + switch (sub.subscriptionState) { + case SUBSCRIBING: + // check timeout + if (currentTime > sub.timeoutValue) { + sub.subscriptionState = SubscriptionState.UNSUBSCRIBED; + sub.timeoutValue = currentTime + sub.backOff; + sub.backOff = sub.backOff * 2; // exponential back of + LOGGER.atDebug().addArgument(sub.device).addArgument(sub.property).log("subscription timed out, retrying: {}/{}"); + } + break; + case UNSUBSCRIBED: + if (currentTime > sub.timeoutValue) { + LOGGER.atDebug().addArgument(sub.device).addArgument(sub.property).log("subscribing {}/{}"); + sendSubscribe(sub); + } + break; + case SUBSCRIBED: + case UNSUBSCRIBE_SENT: + // do nothing + break; + case CANCELED: + sendUnsubscribe(sub); + break; + default: + throw new IllegalStateException("unexpected subscription state: " + sub.subscriptionState); + } + } + + public void sendHeartBeat() { + try { + CmwLightProtocol.sendMsg(socket, CmwLightMessage.CLIENT_HB); + } catch (CmwLightProtocol.RdaLightException e) { + LOGGER.atDebug().setCause(e).log("Error sending heartbeat"); + } + } + + private void sendSubscribe(final Subscription sub) { + if (!sub.subscriptionState.equals(SubscriptionState.UNSUBSCRIBED)) { + return; // already subscribed/subscription in progress + } + try { + CmwLightProtocol.sendMsg(socket, CmwLightMessage.subscribeRequest( + sessionId, sub.id, sub.device, sub.property, + Map.of(CmwLightProtocol.FieldName.SESSION_BODY_TAG.value(), Collections.emptyMap()), + new CmwLightMessage.RequestContext(sub.selector, sub.filters, null), + CmwLightProtocol.UpdateType.IMMEDIATE_UPDATE)); + sub.subscriptionState = SubscriptionState.SUBSCRIBING; + sub.timeoutValue = System.currentTimeMillis() + SUBSCRIPTION_TIMEOUT; + } catch (CmwLightProtocol.RdaLightException e) { + LOGGER.atDebug().setCause(e).log("Error subscribing to property:"); + sub.timeoutValue = System.currentTimeMillis() + sub.backOff; + sub.backOff *= 2; + } + } + + private void sendUnsubscribe(final Subscription sub) { + try { + CmwLightProtocol.sendMsg(socket, CmwLightMessage.unsubscribeRequest( + sessionId, sub.updateId, sub.device, sub.property, + Map.of(CmwLightProtocol.FieldName.SESSION_BODY_TAG.value(), Collections.emptyMap()), + CmwLightProtocol.UpdateType.IMMEDIATE_UPDATE)); + sub.subscriptionState = SubscriptionState.UNSUBSCRIBE_SENT; + } catch (CmwLightProtocol.RdaLightException e) { + LOGGER.atError().addArgument(sub.property).log("failed to unsubscribe "); + } + } + + public static class Subscription { + private final long id = REQUEST_ID_GENERATOR.incrementAndGet(); + public final String property; + public final String device; + public final String selector; + public final Map filters; + public final String endpoint; + public SubscriptionState subscriptionState = SubscriptionState.UNSUBSCRIBED; + public int backOff = 20; + public long updateId = -1; + public long timeoutValue = -1; + public String idString = ""; + + public Subscription(final String endpoint, final String device, final String property, final String selector, final Map filters) { + this.endpoint = endpoint; + this.property = property; + this.device = device; + this.selector = selector; + this.filters = filters; + } + + @Override + public String toString() { + return "Subscription{" + + "property='" + property + '\'' + ", device='" + device + '\'' + ", selector='" + selector + '\'' + ", filters=" + filters + ", subscriptionState=" + subscriptionState + ", backOff=" + backOff + ", id=" + id + ", updateId=" + updateId + ", timeoutValue=" + timeoutValue + '}'; + } + } + + public static class Request { // NOPMD - data class + public final byte[] filters; + public final byte[] data; + public final long id; + private final String requestId; + private final String endpoint; + private final byte[] rbacToken; + public final CmwLightProtocol.RequestType requestType; + + public Request(final CmwLightProtocol.RequestType requestType, + final String requestId, + final String endpoint, + final byte[] filters, // NOPMD - zero copy contract + final byte[] data, // NOPMD - zero copy contract + final byte[] rbacToken // NOPMD - zero copy contract + ) { + this.requestType = requestType; + this.id = REQUEST_ID_GENERATOR.incrementAndGet(); + this.requestId = requestId; + this.endpoint = endpoint; + this.filters = filters; + this.data = data; + this.rbacToken = rbacToken; + } + } + + public enum SubscriptionState { + UNSUBSCRIBED, + SUBSCRIBING, + SUBSCRIBED, + CANCELED, + UNSUBSCRIBE_SENT + } +} diff --git a/client/src/main/java/io/opencmw/client/cmwlight/CmwLightMessage.java b/client/src/main/java/io/opencmw/client/cmwlight/CmwLightMessage.java new file mode 100644 index 00000000..640d498b --- /dev/null +++ b/client/src/main/java/io/opencmw/client/cmwlight/CmwLightMessage.java @@ -0,0 +1,427 @@ +package io.opencmw.client.cmwlight; + +import java.util.Map; +import java.util.Objects; + +import org.zeromq.ZFrame; + +/** + * Data representation for all Messages exchanged between CMW client and server + */ +@SuppressWarnings({ "PMD.ExcessivePublicCount", "PMD.TooManyMethods", "PMD.TooManyFields" }) // - the nature of this class definition +public class CmwLightMessage { + // general fields + public CmwLightProtocol.MessageType messageType; + + // Connection Req/Ack + public String version; + + // header data + public CmwLightProtocol.RequestType requestType; + public long id; + public String deviceName; + public CmwLightProtocol.UpdateType updateType; + public String sessionId; + public String propertyName; + public Map options; + public Map data; + + // additional data + public ZFrame bodyData; + public ExceptionMessage exceptionMessage; + public RequestContext requestContext; + public DataContext dataContext; + + // Subscription Update + public long notificationId; + + // subscription established + public long sourceId; + public Map sessionBody; + + // static instances for low level message types + public static final CmwLightMessage SERVER_HB = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_HB); + public static final CmwLightMessage CLIENT_HB = new CmwLightMessage(CmwLightProtocol.MessageType.CLIENT_HB); + // static functions to get certain message types + public static CmwLightMessage connectAck(final String version) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_CONNECT_ACK); + msg.version = version; + return msg; + } + + public static CmwLightMessage connect(final String version) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.CLIENT_CONNECT); + msg.version = version; + return msg; + } + public static CmwLightMessage subscribeRequest(String sessionId, long id, String device, String property, final Map options, RequestContext requestContext, CmwLightProtocol.UpdateType updateType) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.CLIENT_REQ); + msg.requestType = CmwLightProtocol.RequestType.SUBSCRIBE; + msg.id = id; + msg.options = options; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.requestContext = requestContext; + msg.updateType = updateType; + return msg; + } + public static CmwLightMessage subscribeReply(String sessionId, long id, String device, String property, final Map options) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_REP); + msg.requestType = CmwLightProtocol.RequestType.SUBSCRIBE; + msg.id = id; + msg.options = options; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + return msg; + } + public static CmwLightMessage unsubscribeRequest(String sessionId, long id, String device, String property, final Map options, CmwLightProtocol.UpdateType updateType) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.CLIENT_REQ); + msg.requestType = CmwLightProtocol.RequestType.UNSUBSCRIBE; + msg.id = id; + msg.options = options; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = updateType; + return msg; + } + public static CmwLightMessage getRequest(String sessionId, long id, String device, String property, RequestContext requestContext) { + final CmwLightMessage msg = new CmwLightMessage(); + msg.messageType = CmwLightProtocol.MessageType.CLIENT_REQ; + msg.requestType = CmwLightProtocol.RequestType.GET; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.requestContext = requestContext; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + return msg; + } + + public static CmwLightMessage setRequest(final String sessionId, final long id, final String device, final String property, final ZFrame data, final RequestContext requestContext) { + final CmwLightMessage msg = new CmwLightMessage(); + msg.messageType = CmwLightProtocol.MessageType.CLIENT_REQ; + msg.requestType = CmwLightProtocol.RequestType.SET; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.requestContext = requestContext; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + msg.bodyData = data; + return msg; + } + + public static CmwLightMessage exceptionReply(final String sessionId, final long id, final String device, final String property, final String message, final long contextAcqStamp, final long contextCycleStamp, final byte type) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_REP); + msg.requestType = CmwLightProtocol.RequestType.EXCEPTION; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + msg.exceptionMessage = new ExceptionMessage(contextAcqStamp, contextCycleStamp, message, type); + return msg; + } + + public static CmwLightMessage subscribeExceptionReply(final String sessionId, final long id, final String device, final String property, final String message, final long contextAcqStamp, final long contextCycleStamp, final byte type) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_REP); + msg.requestType = CmwLightProtocol.RequestType.SUBSCRIBE_EXCEPTION; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + msg.exceptionMessage = new ExceptionMessage(contextAcqStamp, contextCycleStamp, message, type); + return msg; + } + + public static CmwLightMessage notificationExceptionReply(final String sessionId, final long id, final String device, final String property, final String message, final long contextAcqStamp, final long contextCycleStamp, final byte type) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_REP); + msg.requestType = CmwLightProtocol.RequestType.NOTIFICATION_EXC; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + msg.exceptionMessage = new ExceptionMessage(contextAcqStamp, contextCycleStamp, message, type); + return msg; + } + + public static CmwLightMessage notificationReply(final String sessionId, final long id, final String device, final String property, final ZFrame data, final long notificationId, final DataContext requestContext, final CmwLightProtocol.UpdateType updateType) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_REP); + msg.requestType = CmwLightProtocol.RequestType.NOTIFICATION_DATA; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.notificationId = notificationId; + msg.options = Map.of(CmwLightProtocol.FieldName.NOTIFICATION_ID_TAG.value(), notificationId); + msg.dataContext = requestContext; + msg.updateType = updateType; + msg.bodyData = data; + return msg; + } + + public static CmwLightMessage getReply(final String sessionId, final long id, final String device, final String property, final ZFrame data, final DataContext requestContext) { + final CmwLightMessage msg = new CmwLightMessage(CmwLightProtocol.MessageType.SERVER_REP); + msg.requestType = CmwLightProtocol.RequestType.REPLY; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.dataContext = requestContext; + msg.bodyData = data; + return msg; + } + + public static CmwLightMessage sessionConfirmReply(final String sessionId, final long id, final String device, final String property, final Map options) { + final CmwLightMessage msg = new CmwLightMessage(); + msg.messageType = CmwLightProtocol.MessageType.SERVER_REP; + msg.requestType = CmwLightProtocol.RequestType.SESSION_CONFIRM; + msg.id = id; + msg.options = options; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + return msg; + } + + public static CmwLightMessage eventReply(final String sessionId, final long id, final String device, final String property) { + final CmwLightMessage msg = new CmwLightMessage(); + msg.messageType = CmwLightProtocol.MessageType.SERVER_REP; + msg.requestType = CmwLightProtocol.RequestType.EVENT; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + return msg; + } + + public static CmwLightMessage eventRequest(final String sessionId, final long id, final String device, final String property) { + final CmwLightMessage msg = new CmwLightMessage(); + msg.messageType = CmwLightProtocol.MessageType.CLIENT_REQ; + msg.requestType = CmwLightProtocol.RequestType.EVENT; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + return msg; + } + + public static CmwLightMessage connectRequest(final String sessionId, final long id, final String device, final String property) { + final CmwLightMessage msg = new CmwLightMessage(); + msg.messageType = CmwLightProtocol.MessageType.CLIENT_REQ; + msg.requestType = CmwLightProtocol.RequestType.CONNECT; + msg.id = id; + msg.sessionId = sessionId; + msg.deviceName = device; + msg.propertyName = property; + msg.updateType = CmwLightProtocol.UpdateType.NORMAL; + return msg; + } + + protected CmwLightMessage() { + // Constructor only accessible from within serialiser and factory methods to only allow valid messages + } + + protected CmwLightMessage(final CmwLightProtocol.MessageType messageType) { + this.messageType = messageType; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CmwLightMessage)) { + return false; + } + final CmwLightMessage that = (CmwLightMessage) o; + return id == that.id && notificationId == that.notificationId && sourceId == that.sourceId && messageType == that.messageType && Objects.equals(version, that.version) && requestType == that.requestType && Objects.equals(deviceName, that.deviceName) && updateType == that.updateType && Objects.equals(sessionId, that.sessionId) && Objects.equals(propertyName, that.propertyName) && Objects.equals(options, that.options) && Objects.equals(data, that.data) && Objects.equals(bodyData, that.bodyData) && Objects.equals(exceptionMessage, that.exceptionMessage) && Objects.equals(requestContext, that.requestContext) && Objects.equals(dataContext, that.dataContext) && Objects.equals(sessionBody, that.sessionBody); + } + + @Override + public int hashCode() { + return Objects.hash(messageType, version, requestType, id, deviceName, updateType, sessionId, propertyName, options, data, bodyData, exceptionMessage, requestContext, dataContext, notificationId, sourceId, sessionBody); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CmwMessage: "); + switch (messageType) { + case CLIENT_CONNECT: + sb.append("Connection request, client version='").append(version).append('\''); + break; + case SERVER_CONNECT_ACK: + sb.append("Connection ack, server version='").append(version).append('\''); + break; + case CLIENT_HB: + sb.append("client heartbeat"); + break; + case SERVER_HB: + sb.append("server heartbeat"); + break; + case SERVER_REP: + sb.append("server reply: ").append(requestType.name()); + case CLIENT_REQ: + if (messageType == CmwLightProtocol.MessageType.CLIENT_REQ) { + sb.append("client request: ").append(requestType.name()); + } + sb.append(" id: ").append(id).append(" deviceName=").append(deviceName).append(", updateType=").append(updateType).append(", sessionId='").append(sessionId).append("', propertyName='").append(propertyName).append("', options=").append(options).append(", data=").append(data).append(", sourceId=").append(sourceId); + switch (requestType) { + case GET: + case SET: + case SUBSCRIBE: + case UNSUBSCRIBE: + sb.append("\n requestContext=").append(requestContext); + break; + case REPLY: + case NOTIFICATION_DATA: + sb.append(", notificationId=").append(notificationId).append("\n bodyData=").append(bodyData).append("\n dataContext=").append(dataContext); + break; + case EXCEPTION: + case NOTIFICATION_EXC: + case SUBSCRIBE_EXCEPTION: + sb.append("\n exceptionMessage=").append(exceptionMessage); + break; + case SESSION_CONFIRM: + sb.append(", sessionBody='").append(sessionBody).append('\''); + break; + case CONNECT: + case EVENT: + break; + default: + throw new IllegalStateException("unknown client request message type: " + messageType); + } + break; + default: + throw new IllegalStateException("unknown message type: " + messageType); + } + return sb.toString(); + } + + public static class RequestContext { + public String selector; + public Map data; + public Map filters; + + public RequestContext(final String selector, final Map filters, final Map data) { + this.selector = selector; + this.filters = filters; + this.data = data; + } + + protected RequestContext() { + // default constructor only available to protocol (de)serialisers + } + + @Override + public String toString() { + return "RequestContext{" + + "selector='" + selector + '\'' + ", data=" + data + ", filters=" + filters + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RequestContext)) { + return false; + } + final RequestContext that = (RequestContext) o; + return selector.equals(that.selector) && Objects.equals(data, that.data) && Objects.equals(filters, that.filters); + } + + @Override + public int hashCode() { + return Objects.hash(selector, data, filters); + } + } + + public static class DataContext { + public String cycleName; + public long cycleStamp; + public long acqStamp; + public Map data; + + public DataContext(final String cycleName, final long cycleStamp, final long acqStamp, final Map data) { + this.cycleName = cycleName; + this.cycleStamp = cycleStamp; + this.acqStamp = acqStamp; + this.data = data; + } + + protected DataContext() { + // allow only protocol serialiser to create empty object + } + + @Override + public String toString() { + return "DataContext{cycleName='" + cycleName + '\'' + ", cycleStamp=" + cycleStamp + ", acqStamp=" + acqStamp + ", data=" + data + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DataContext)) { + return false; + } + final DataContext that = (DataContext) o; + return cycleStamp == that.cycleStamp && acqStamp == that.acqStamp && cycleName.equals(that.cycleName) && Objects.equals(data, that.data); + } + + @Override + public int hashCode() { + return Objects.hash(cycleName, cycleStamp, acqStamp, data); + } + } + + public static class ExceptionMessage { + public long contextAcqStamp; + public long contextCycleStamp; + public String message; + public byte type; + + public ExceptionMessage(final long contextAcqStamp, final long contextCycleStamp, final String message, final byte type) { + this.contextAcqStamp = contextAcqStamp; + this.contextCycleStamp = contextCycleStamp; + this.message = message; + this.type = type; + } + + protected ExceptionMessage() { + // allow only protocol serialiser to create empty object + } + + @Override + public String toString() { + return "ExceptionMessage{contextAcqStamp=" + contextAcqStamp + ", contextCycleStamp=" + contextCycleStamp + ", message='" + message + '\'' + ", type=" + type + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ExceptionMessage)) { + return false; + } + final ExceptionMessage that = (ExceptionMessage) o; + return contextAcqStamp == that.contextAcqStamp && contextCycleStamp == that.contextCycleStamp && type == that.type && message.equals(that.message); + } + + @Override + public int hashCode() { + return Objects.hash(contextAcqStamp, contextCycleStamp, message, type); + } + } +} diff --git a/client/src/main/java/io/opencmw/client/cmwlight/CmwLightProtocol.java b/client/src/main/java/io/opencmw/client/cmwlight/CmwLightProtocol.java new file mode 100644 index 00000000..e42ba502 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/cmwlight/CmwLightProtocol.java @@ -0,0 +1,621 @@ +package io.opencmw.client.cmwlight; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.jetbrains.annotations.NotNull; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +/** + * A lightweight implementation of the CMW RDA client protocol part. + * Serializes CmwLightMessage to ZeroMQ messages and vice versa. + */ +@SuppressWarnings("PMD.UnusedLocalVariable") // Unused variables are taken from the protocol and should be available for reference +public class CmwLightProtocol { //NOPMD -- nomen est omen + private static final String CONTEXT_ACQ_STAMP = "ContextAcqStamp"; + private static final String CONTEXT_CYCLE_STAMP = "ContextCycleStamp"; + private static final String MESSAGE = "Message"; + private static final String TYPE = "Type"; + private static final String EMPTY_CONTEXT = "empty context data for request type: "; + private static final int MAX_MSG_SIZE = 4096 * 1024; + private static final IoBuffer IO_BUFFER = new FastByteBuffer(MAX_MSG_SIZE); + private static final CmwLightSerialiser SERIALISER = new CmwLightSerialiser(IO_BUFFER); + private static final IoClassSerialiser IO_CLASS_SERIALISER = new IoClassSerialiser(IO_BUFFER); + public static final String VERSION = "1.0.0"; // Protocol version used if msg.version is null or empty + private static final int SERIALISER_QUIRK = 100; // there seems to be a bug in the serialiser which does not update the buffer position correctly, so send more + + private CmwLightProtocol() { + // utility class + } + + /** + * The message specified by the byte contained in the first frame of a message defines what type of message is present + */ + public enum MessageType { + SERVER_CONNECT_ACK(0x01), + SERVER_REP(0x02), + SERVER_HB(0x03), + CLIENT_CONNECT(0x20), + CLIENT_REQ(0x21), + CLIENT_HB(0x22); + + private static final int CLIENT_API_RANGE = 0x4; + private static final int SERVER_API_RANGE = 0x20; + private final byte value; + + MessageType(int value) { + this.value = (byte) value; + } + + public byte value() { + return value; + } + + public static MessageType of(int value) { // NOPMD -- nomen est omen + if (value < CLIENT_API_RANGE) { + return values()[value - 1]; + } else { + return values()[value - SERVER_API_RANGE + CLIENT_CONNECT.ordinal()]; + } + } + } + + /** + * Frame Types in the descriptor (Last frame of a message containing the type of each sub message) + */ + public enum FrameType { + HEADER(0), + BODY(1), + BODY_DATA_CONTEXT(2), + BODY_REQUEST_CONTEXT(3), + BODY_EXCEPTION(4); + + private final byte value; + + FrameType(int value) { + this.value = (byte) value; + } + + public byte value() { + return value; + } + } + + /** + * Field names for the Request Header + */ + public enum FieldName { + EVENT_TYPE_TAG("eventType"), + MESSAGE_TAG("message"), + ID_TAG("0"), + DEVICE_NAME_TAG("1"), + REQ_TYPE_TAG("2"), + OPTIONS_TAG("3"), + CYCLE_NAME_TAG("4"), + ACQ_STAMP_TAG("5"), + CYCLE_STAMP_TAG("6"), + UPDATE_TYPE_TAG("7"), + SELECTOR_TAG("8"), + CLIENT_INFO_TAG("9"), + NOTIFICATION_ID_TAG("a"), + SOURCE_ID_TAG("b"), + FILTERS_TAG("c"), + DATA_TAG("x"), + SESSION_ID_TAG("d"), + SESSION_BODY_TAG("e"), + PROPERTY_NAME_TAG("f"); + + private final String name; + + FieldName(String name) { + this.name = name; + } + + public String value() { + return name; + } + } + + /** + * request type used in request header REQ_TYPE_TAG + */ + public enum RequestType { + GET(0), + SET(1), + CONNECT(2), + REPLY(3), + EXCEPTION(4), + SUBSCRIBE(5), + UNSUBSCRIBE(6), + NOTIFICATION_DATA(7), + NOTIFICATION_EXC(8), + SUBSCRIBE_EXCEPTION(9), + EVENT(10), + SESSION_CONFIRM(11); + + private final byte value; + + RequestType(int value) { + this.value = (byte) value; + } + + public static RequestType of(int value) { // NOPMD - nomen est omen + return values()[value]; + } + + public byte value() { + return value; + } + } + + /** + * UpdateType + */ + public enum UpdateType { + NORMAL(0), + FIRST_UPDATE(1), // Initial update sent when the subscription is created. + IMMEDIATE_UPDATE(2); // Update sent after the value has been modified by a set call. + + private final byte value; + + UpdateType(int value) { + this.value = (byte) value; + } + + public static UpdateType of(int value) { // NOPMD - nomen est omen + return values()[value]; + } + + public byte value() { + return value; + } + } + + public static CmwLightMessage recvMsg(final ZMQ.Socket socket, int tout) throws RdaLightException { + return parseMsg(ZMsg.recvMsg(socket, tout)); + } + + public static CmwLightMessage parseMsg(final @NotNull ZMsg data) throws RdaLightException { // NOPMD - NPath complexity acceptable (complex protocol) + assert data != null : "data"; + final ZFrame firstFrame = data.pollFirst(); + if (firstFrame != null && Arrays.equals(firstFrame.getData(), new byte[] { MessageType.SERVER_CONNECT_ACK.value() })) { + final CmwLightMessage reply = new CmwLightMessage(MessageType.SERVER_CONNECT_ACK); + final ZFrame versionData = data.pollFirst(); + assert versionData != null : "version data in connection acknowledgement frame"; + reply.version = versionData.getString(Charset.defaultCharset()); + return reply; + } + if (firstFrame != null && Arrays.equals(firstFrame.getData(), new byte[] { MessageType.CLIENT_CONNECT.value() })) { + final CmwLightMessage reply = new CmwLightMessage(MessageType.CLIENT_CONNECT); + final ZFrame versionData = data.pollFirst(); + assert versionData != null : "version data in connection acknowledgement frame"; + reply.version = versionData.getString(Charset.defaultCharset()); + return reply; + } + if (firstFrame != null && Arrays.equals(firstFrame.getData(), new byte[] { MessageType.SERVER_HB.value() })) { + return CmwLightMessage.SERVER_HB; + } + if (firstFrame != null && Arrays.equals(firstFrame.getData(), new byte[] { MessageType.CLIENT_HB.value() })) { + return CmwLightMessage.CLIENT_HB; + } + byte[] descriptor = checkDescriptor(data.pollLast(), firstFrame); + final ZFrame headerMsg = data.poll(); + assert headerMsg != null : "message header"; + CmwLightMessage reply = getReplyFromHeader(firstFrame, headerMsg); + switch (reply.requestType) { + case REPLY: + assertDescriptor(descriptor, FrameType.HEADER, FrameType.BODY, FrameType.BODY_DATA_CONTEXT); + reply.bodyData = data.pollFirst(); + if (data.isEmpty()) { + throw new RdaLightException(EMPTY_CONTEXT + reply.requestType); + } + reply.dataContext = parseContextData(data.pollFirst()); + return reply; + case NOTIFICATION_DATA: // notification update + assertDescriptor(descriptor, FrameType.HEADER, FrameType.BODY, FrameType.BODY_DATA_CONTEXT); + if (reply.options != null && reply.options.containsKey(FieldName.NOTIFICATION_ID_TAG.value())) { + reply.notificationId = (long) reply.options.get(FieldName.NOTIFICATION_ID_TAG.value()); + } + reply.bodyData = data.pollFirst(); + if (data.isEmpty()) { + throw new RdaLightException(EMPTY_CONTEXT + reply.requestType); + } + reply.dataContext = parseContextData(data.pollFirst()); + return reply; + case EXCEPTION: // exception on get/set request + case NOTIFICATION_EXC: // exception on notification, e.g null pointer in server notify code + case SUBSCRIBE_EXCEPTION: // exception on subscribe e.g. nonexistent property, wrong filters + assertDescriptor(descriptor, FrameType.HEADER, FrameType.BODY_EXCEPTION); + reply.exceptionMessage = parseExceptionMessage(data.pollFirst()); + return reply; + case GET: + assertDescriptor(descriptor, FrameType.HEADER, FrameType.BODY_REQUEST_CONTEXT); + if (data.isEmpty()) { + throw new RdaLightException(EMPTY_CONTEXT + reply.requestType); + } + reply.requestContext = parseRequestContext(data.pollFirst()); + return reply; + case SUBSCRIBE: // descriptor: [0] options: SOURCE_ID_TAG // seems to be sent after subscription is accepted + if (reply.messageType == MessageType.SERVER_REP) { + assertDescriptor(descriptor, FrameType.HEADER); + if (reply.options != null && reply.options.containsKey(FieldName.SOURCE_ID_TAG.value())) { + reply.sourceId = (long) reply.options.get(FieldName.SOURCE_ID_TAG.value()); + } + } else { + assertDescriptor(descriptor, FrameType.HEADER, FrameType.BODY_REQUEST_CONTEXT); + if (data.isEmpty()) { + throw new RdaLightException(EMPTY_CONTEXT + reply.requestType); + } + reply.requestContext = parseRequestContext(data.pollFirst()); + } + return reply; + case SESSION_CONFIRM: // descriptor: [0] options: SESSION_BODY_TAG + assertDescriptor(descriptor, FrameType.HEADER); + if (reply.options != null && reply.options.containsKey(FieldName.SESSION_BODY_TAG.value())) { + final Object subMap = reply.options.get(FieldName.SESSION_BODY_TAG.value()); + final String fieldName = FieldName.SESSION_BODY_TAG.value(); + if (subMap instanceof Map) { + @SuppressWarnings("unchecked") + final Map castMap = (Map) reply.options.get(fieldName); + reply.sessionBody = castMap; + } else { + throw new RdaLightException("field member '" + fieldName + "' not assignable to Map: " + subMap); + } + } + return reply; + case EVENT: + case UNSUBSCRIBE: + case CONNECT: + assertDescriptor(descriptor, FrameType.HEADER); + return reply; + case SET: + assertDescriptor(descriptor, FrameType.HEADER, FrameType.BODY, FrameType.BODY_REQUEST_CONTEXT); + reply.bodyData = data.pollFirst(); + if (data.isEmpty()) { + throw new RdaLightException(EMPTY_CONTEXT + reply.requestType); + } + reply.requestContext = parseRequestContext(data.pollFirst()); + return reply; + default: + throw new RdaLightException("received unknown or non-client request type: " + reply.requestType); + } + } + + public static void sendMsg(final ZMQ.Socket socket, final CmwLightMessage msg) throws RdaLightException { + serialiseMsg(msg).send(socket); + } + + public static ZMsg serialiseMsg(final CmwLightMessage msg) throws RdaLightException { + final ZMsg result = new ZMsg(); + switch (msg.messageType) { + case SERVER_CONNECT_ACK: + case CLIENT_CONNECT: + result.add(new ZFrame(new byte[] { msg.messageType.value() })); + result.add(new ZFrame(msg.version == null || msg.version.isEmpty() ? VERSION : msg.version)); + return result; + case CLIENT_HB: + case SERVER_HB: + result.add(new ZFrame(new byte[] { msg.messageType.value() })); + return result; + case SERVER_REP: + case CLIENT_REQ: + result.add(new byte[] { msg.messageType.value() }); + result.add(serialiseHeader(msg)); + switch (msg.requestType) { + case CONNECT: + case EVENT: + case SESSION_CONFIRM: + case UNSUBSCRIBE: + addDescriptor(result, FrameType.HEADER); + break; + case GET: + case SUBSCRIBE: + if (msg.messageType == MessageType.CLIENT_REQ) { + assert msg.requestContext != null : "requestContext"; + result.add(serialiseRequestContext(msg.requestContext)); + addDescriptor(result, FrameType.HEADER, FrameType.BODY_REQUEST_CONTEXT); + } else { + addDescriptor(result, FrameType.HEADER); + } + break; + case SET: + assert msg.bodyData != null : "bodyData"; + assert msg.requestContext != null : "requestContext"; + result.add(msg.bodyData); + result.add(serialiseRequestContext(msg.requestContext)); + addDescriptor(result, FrameType.HEADER, FrameType.BODY, FrameType.BODY_REQUEST_CONTEXT); + break; + case REPLY: + case NOTIFICATION_DATA: + assert msg.bodyData != null : "bodyData"; + result.add(msg.bodyData); + result.add(serialiseDataContext(msg.dataContext)); + addDescriptor(result, FrameType.HEADER, FrameType.BODY, FrameType.BODY_DATA_CONTEXT); + break; + case NOTIFICATION_EXC: + case EXCEPTION: + case SUBSCRIBE_EXCEPTION: + assert msg.exceptionMessage != null : "exceptionMessage"; + result.add(serialiseExceptionMessage(msg.exceptionMessage)); + addDescriptor(result, FrameType.HEADER, FrameType.BODY_EXCEPTION); + break; + default: + } + return result; + default: + } + + throw new RdaLightException("Invalid cmwMessage: " + msg); + } + + private static ZFrame serialiseExceptionMessage(final CmwLightMessage.ExceptionMessage exceptionMessage) { + IO_BUFFER.reset(); + SERIALISER.setBuffer(IO_BUFFER); + SERIALISER.putHeaderInfo(); + SERIALISER.put(CONTEXT_ACQ_STAMP, exceptionMessage.contextAcqStamp); + SERIALISER.put(CONTEXT_CYCLE_STAMP, exceptionMessage.contextCycleStamp); + SERIALISER.put(MESSAGE, exceptionMessage.message); + SERIALISER.put(TYPE, exceptionMessage.type); + IO_BUFFER.flip(); + return new ZFrame(Arrays.copyOfRange(IO_BUFFER.elements(), 0, IO_BUFFER.limit() + SERIALISER_QUIRK)); + } + + private static void addDescriptor(final ZMsg result, final FrameType... frametypes) { + byte[] descriptor = new byte[frametypes.length]; + for (int i = 0; i < descriptor.length; i++) { + descriptor[i] = frametypes[i].value(); + } + result.add(new ZFrame(descriptor)); + } + + private static ZFrame serialiseHeader(final CmwLightMessage msg) throws RdaLightException { + IO_BUFFER.reset(); + SERIALISER.setBuffer(IO_BUFFER); + SERIALISER.putHeaderInfo(); + SERIALISER.put(FieldName.REQ_TYPE_TAG.value(), msg.requestType.value()); + SERIALISER.put(FieldName.ID_TAG.value(), msg.id); + SERIALISER.put(FieldName.DEVICE_NAME_TAG.value(), msg.deviceName); + SERIALISER.put(FieldName.PROPERTY_NAME_TAG.value(), msg.propertyName); + if (msg.updateType != null) { + SERIALISER.put(FieldName.UPDATE_TYPE_TAG.value(), msg.updateType.value()); + } + SERIALISER.put(FieldName.SESSION_ID_TAG.value(), msg.sessionId); + // StartMarker marks start of Data Object + putMap(SERIALISER, FieldName.OPTIONS_TAG.value(), msg.options); + IO_BUFFER.flip(); + return new ZFrame(Arrays.copyOfRange(IO_BUFFER.elements(), 0, IO_BUFFER.limit() + SERIALISER_QUIRK)); + } + + private static ZFrame serialiseRequestContext(final CmwLightMessage.RequestContext requestContext) throws RdaLightException { + IO_BUFFER.reset(); + SERIALISER.putHeaderInfo(); + SERIALISER.put(FieldName.SELECTOR_TAG.value(), requestContext.selector); + putMap(SERIALISER, FieldName.FILTERS_TAG.value(), requestContext.filters); + putMap(SERIALISER, FieldName.DATA_TAG.value(), requestContext.data); + IO_BUFFER.flip(); + return new ZFrame(Arrays.copyOfRange(IO_BUFFER.elements(), 0, IO_BUFFER.limit() + SERIALISER_QUIRK)); + } + + private static ZFrame serialiseDataContext(final CmwLightMessage.DataContext dataContext) throws RdaLightException { + IO_BUFFER.reset(); + SERIALISER.putHeaderInfo(); + SERIALISER.put(FieldName.CYCLE_NAME_TAG.value(), dataContext.cycleName); + SERIALISER.put(FieldName.CYCLE_STAMP_TAG.value(), dataContext.cycleStamp); + SERIALISER.put(FieldName.ACQ_STAMP_TAG.value(), dataContext.acqStamp); + putMap(SERIALISER, FieldName.DATA_TAG.value(), dataContext.data); + IO_BUFFER.flip(); + return new ZFrame(Arrays.copyOfRange(IO_BUFFER.elements(), 0, IO_BUFFER.limit() + SERIALISER_QUIRK)); + } + + private static void putMap(final CmwLightSerialiser serialiser, final String fieldName, final Map map) throws RdaLightException { + if (map != null) { + final WireDataFieldDescription dataFieldMarker = new WireDataFieldDescription(serialiser, serialiser.getParent(), -1, + fieldName, DataType.START_MARKER, -1, -1, -1); + serialiser.putStartMarker(dataFieldMarker); + for (final Map.Entry entry : map.entrySet()) { + if (entry.getValue() instanceof String) { + serialiser.put(entry.getKey(), (String) entry.getValue()); + } else if (entry.getValue() instanceof Integer) { + serialiser.put(entry.getKey(), (Integer) entry.getValue()); + } else if (entry.getValue() instanceof Long) { + serialiser.put(entry.getKey(), (Long) entry.getValue()); + } else if (entry.getValue() instanceof Boolean) { + serialiser.put(entry.getKey(), (Boolean) entry.getValue()); + } else if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + final Map subMap = (Map) entry.getValue(); + putMap(serialiser, entry.getKey(), subMap); + } else { + throw new RdaLightException("unsupported map entry type: " + entry.getValue().getClass().getCanonicalName()); + } + } + serialiser.putEndMarker(dataFieldMarker); + } + } + + private static CmwLightMessage getReplyFromHeader(final ZFrame firstFrame, final ZFrame header) throws RdaLightException { + CmwLightMessage reply = new CmwLightMessage(MessageType.of(firstFrame.getData()[0])); + IO_CLASS_SERIALISER.setDataBuffer(FastByteBuffer.wrap(header.getData())); + final FieldDescription headerMap; + try { + headerMap = IO_CLASS_SERIALISER.parseWireFormat().getChildren().get(0); + for (FieldDescription field : headerMap.getChildren()) { + if (field.getFieldName().equals(FieldName.REQ_TYPE_TAG.value()) && field.getType() == byte.class) { + reply.requestType = RequestType.of((byte) (((WireDataFieldDescription) field).data())); + } else if (field.getFieldName().equals(FieldName.ID_TAG.value()) && field.getType() == long.class) { + reply.id = (long) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.DEVICE_NAME_TAG.value()) && field.getType() == String.class) { + reply.deviceName = (String) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.OPTIONS_TAG.value())) { + reply.options = readMap(field); + } else if (field.getFieldName().equals(FieldName.UPDATE_TYPE_TAG.value()) && field.getType() == byte.class) { + reply.updateType = UpdateType.of((byte) ((WireDataFieldDescription) field).data()); + } else if (field.getFieldName().equals(FieldName.SESSION_ID_TAG.value()) && field.getType() == String.class) { + reply.sessionId = (String) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.PROPERTY_NAME_TAG.value()) && field.getType() == String.class) { + reply.propertyName = (String) ((WireDataFieldDescription) field).data(); + } else { + throw new RdaLightException("Unknown CMW header field: " + field.getFieldName()); + } + } + } catch (IllegalStateException e) { + throw new RdaLightException("unparsable header: " + Arrays.toString(header.getData()) + "(" + header.toString() + ")", e); + } + if (reply.requestType == null) { + throw new RdaLightException("Header does not contain request type field"); + } + return reply; + } + + private static Map readMap(final FieldDescription field) { + Map result = null; + for (FieldDescription dataField : field.getChildren()) { + if (result == null) { + result = new HashMap<>(); // NOPMD - necessary to allocate inside loop + } + //if ( 'condition' ) { + // find out how to see if the field is itself a map + // result.put(dataField.getFieldName(), readMap(dataField)) + // } else { + result.put(dataField.getFieldName(), ((WireDataFieldDescription) dataField).data()); + //} + } + return result; + } + + private static CmwLightMessage.ExceptionMessage parseExceptionMessage(final ZFrame exceptionBody) throws RdaLightException { + if (exceptionBody == null) { + throw new RdaLightException("malformed subscription exception"); + } + final CmwLightMessage.ExceptionMessage exceptionMessage = new CmwLightMessage.ExceptionMessage(); + IO_CLASS_SERIALISER.setDataBuffer(FastByteBuffer.wrap(exceptionBody.getData())); + final FieldDescription exceptionFields = IO_CLASS_SERIALISER.parseWireFormat().getChildren().get(0); + for (FieldDescription field : exceptionFields.getChildren()) { + if (CONTEXT_ACQ_STAMP.equals(field.getFieldName()) && field.getType() == long.class) { + exceptionMessage.contextAcqStamp = (long) ((WireDataFieldDescription) field).data(); + } else if (CONTEXT_CYCLE_STAMP.equals(field.getFieldName()) && field.getType() == long.class) { + exceptionMessage.contextCycleStamp = (long) ((WireDataFieldDescription) field).data(); + } else if (MESSAGE.equals(field.getFieldName()) && field.getType() == String.class) { + exceptionMessage.message = (String) ((WireDataFieldDescription) field).data(); + } else if (TYPE.equals(field.getFieldName()) && field.getType() == byte.class) { + exceptionMessage.type = (byte) ((WireDataFieldDescription) field).data(); + } else { + throw new RdaLightException("Unsupported field in exception body: " + field.getFieldName()); + } + } + return exceptionMessage; + } + + private static CmwLightMessage.RequestContext parseRequestContext(final @NotNull ZFrame contextData) throws RdaLightException { + assert contextData != null : "contextData"; + CmwLightMessage.RequestContext requestContext = new CmwLightMessage.RequestContext(); + IO_CLASS_SERIALISER.setDataBuffer(FastByteBuffer.wrap(contextData.getData())); + final FieldDescription contextMap; + try { + contextMap = IO_CLASS_SERIALISER.parseWireFormat().getChildren().get(0); + for (FieldDescription field : contextMap.getChildren()) { + if (field.getFieldName().equals(FieldName.SELECTOR_TAG.value()) && field.getType() == String.class) { + requestContext.selector = (String) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.FILTERS_TAG.value())) { + for (FieldDescription dataField : field.getChildren()) { + if (requestContext.filters == null) { + requestContext.filters = new HashMap<>(); // NOPMD - necessary to allocate inside loop + } + requestContext.filters.put(dataField.getFieldName(), ((WireDataFieldDescription) dataField).data()); + } + } else if (field.getFieldName().equals(FieldName.DATA_TAG.value())) { + for (FieldDescription dataField : field.getChildren()) { + if (requestContext.data == null) { + requestContext.data = new HashMap<>(); // NOPMD - necessary to allocate inside loop + } + requestContext.data.put(dataField.getFieldName(), ((WireDataFieldDescription) dataField).data()); + } + } else { + throw new UnsupportedOperationException("Unknown field: " + field.getFieldName()); + } + } + } catch (IllegalStateException e) { + throw new RdaLightException("unparsable context data: " + Arrays.toString(contextData.getData()) + "(" + new String(contextData.getData()) + ")", e); + } + return requestContext; + } + + private static CmwLightMessage.DataContext parseContextData(final @NotNull ZFrame contextData) throws RdaLightException { + assert contextData != null : "contextData"; + CmwLightMessage.DataContext dataContext = new CmwLightMessage.DataContext(); + IO_CLASS_SERIALISER.setDataBuffer(FastByteBuffer.wrap(contextData.getData())); + final FieldDescription contextMap; + try { + contextMap = IO_CLASS_SERIALISER.parseWireFormat().getChildren().get(0); + for (FieldDescription field : contextMap.getChildren()) { + if (field.getFieldName().equals(FieldName.CYCLE_NAME_TAG.value()) && field.getType() == String.class) { + dataContext.cycleName = (String) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.ACQ_STAMP_TAG.value()) && field.getType() == long.class) { + dataContext.acqStamp = (long) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.CYCLE_STAMP_TAG.value()) && field.getType() == long.class) { + dataContext.cycleStamp = (long) ((WireDataFieldDescription) field).data(); + } else if (field.getFieldName().equals(FieldName.DATA_TAG.value())) { + for (FieldDescription dataField : field.getChildren()) { + if (dataContext.data == null) { + dataContext.data = new HashMap<>(); // NOPMD - necessary to allocate inside loop + } + dataContext.data.put(dataField.getFieldName(), ((WireDataFieldDescription) dataField).data()); + } + } else { + throw new UnsupportedOperationException("Unknown field: " + field.getFieldName()); + } + } + } catch (IllegalStateException e) { + throw new RdaLightException("unparsable context data: " + Arrays.toString(contextData.getData()) + "(" + new String(contextData.getData()) + ")", e); + } + return dataContext; + } + + private static void assertDescriptor(final byte[] descriptor, final FrameType... frameTypes) throws RdaLightException { + if (descriptor.length != frameTypes.length) { + throw new RdaLightException("descriptor does not match message type: \n " + Arrays.toString(descriptor) + "\n " + Arrays.toString(frameTypes)); + } + for (int i = 1; i < descriptor.length; i++) { + if (descriptor[i] != frameTypes[i].value()) { + throw new RdaLightException("descriptor does not match message type: \n " + Arrays.toString(descriptor) + "\n " + Arrays.toString(frameTypes)); + } + } + } + + private static byte[] checkDescriptor(final ZFrame descriptorMsg, final ZFrame firstFrame) throws RdaLightException { + if (firstFrame == null || !(Arrays.equals(firstFrame.getData(), new byte[] { MessageType.SERVER_REP.value() }) || Arrays.equals(firstFrame.getData(), new byte[] { MessageType.CLIENT_REQ.value() }))) { + throw new RdaLightException("Expecting only messages of type Heartbeat or Reply but got: " + firstFrame); + } + if (descriptorMsg == null) { + throw new RdaLightException("Message does not contain descriptor"); + } + final byte[] descriptor = descriptorMsg.getData(); + if (descriptor[0] != FrameType.HEADER.value()) { + throw new RdaLightException("First message of SERVER_REP has to be of type MT_HEADER but is: " + descriptor[0]); + } + return descriptor; + } + + public static class RdaLightException extends Exception { + private static final long serialVersionUID = 5197623305559702319L; + public RdaLightException(final String msg) { + super(msg); + } + + public RdaLightException(final String msg, final Throwable e) { + super(msg, e); + } + } +} diff --git a/client/src/main/java/io/opencmw/client/cmwlight/DirectoryLightClient.java b/client/src/main/java/io/opencmw/client/cmwlight/DirectoryLightClient.java new file mode 100644 index 00000000..5d8e0f12 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/cmwlight/DirectoryLightClient.java @@ -0,0 +1,203 @@ +package io.opencmw.client.cmwlight; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URLDecoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Obtain device info from the directory server + */ +public class DirectoryLightClient { + public static final String GET_DEVICE_INFO = "get-device-info"; + // public static final String GET_SERVER_INFO = "get-server-info"; + // private static final String SUPPORTED_CHARACTERS = "\\.\\-\\+_a-zA-Z0-9"; + // private static final String NAME_REGEX = "[a-zA-Z0-9][" + SUPPORTED_CHARACTERS + "]*"; + // private static final String CLIENT_INFO_SUPPORTED_CHARACTERS = "\\x20-\\x7E"; // ASCII := {32-126} + private static final String ERROR_STRING = "ERROR"; + private static final String HOST_PORT_SEPARATOR = ":"; + + private static final String NOT_BOUND_LOCATION = "*NOT_BOUND*"; + // static final String UNKNOWN_SERVER = "*UNKNOWN*"; + private static final String CLIENT_INFO = "DirectoryLightClient"; + private static final String VERSION = "2.0.0"; + private final String nameserver; + private final int nameserverPort; + + public DirectoryLightClient(final String... nameservers) throws DirectoryClientException { + if (nameservers.length != 1) { + throw new DirectoryClientException("only one nameserver supported at the moment"); + } + final String[] hostport = nameservers[0].split(HOST_PORT_SEPARATOR); + if (hostport.length != 2) { + throw new DirectoryClientException("nameserver address has wrong format: " + nameservers[0]); + } + nameserver = hostport[0]; + nameserverPort = Integer.parseInt(hostport[1]); + } + + /** + * Build the request message to query a number of devices + * + * @param devices The devices to query information for + * @return The request message to send to the server + **/ + private String getDeviceMsg(final List devices) { + final StringBuilder sb = new StringBuilder(); + sb.append(GET_DEVICE_INFO).append("\n@client-info ").append(CLIENT_INFO).append("\n@version ").append(VERSION).append('\n'); + // msg.append("@prefer-proxy\n"); + // msg.append("@direct ").append(this.properties.directServers.getValue()).append("\n"); + // msg.append("@domain "); + // for (Domain domain : domains) { + // msg.append(domain.getName()); + // msg.append(","); + // } + // msg.deleteCharAt(msg.length()-1); + // msg.append("\n"); + for (final String dev : devices) { + sb.append(dev).append('\n'); + } + sb.append('\n'); + return sb.toString(); + } + + // /** + // * Build the request message to query a number of servers + // * + // * @param servers The servers to query information for + // * @return The request message to send to the server + // **/ + // private String getServerMsg(final List servers) { + // final StringBuilder sb = new StringBuilder(); + // sb.append(GET_SERVER_INFO).append("\n"); + // sb.append("@client-info ").append(CLIENT_INFO).append("\n"); + // sb.append("@version ").append(VERSION).append("\n"); + // // msg.append("@prefer-proxy\n"); + // // msg.append("@direct ").append(this.properties.directServers.getValue()).append("\n"); + // // msg.append("@domain "); + // // for (Domain domain : domains) { + // // msg.append(domain.getName()); + // // msg.append(","); + // // } + // // msg.deleteCharAt(msg.length()-1); + // // msg.append("\n"); + // for (final String dev : servers) { + // sb.append(dev).append('\n'); + // } + // sb.append('\n'); + // return sb.toString(); + // } + + /** + * Query Server information for a given list of devices. + * + * @param devices The devices to query information for + * @return a list of device information for the queried devices + **/ + public List getDeviceInfo(final List devices) throws DirectoryClientException { + final ArrayList result = new ArrayList<>(); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(nameserver, nameserverPort)); + try (PrintWriter writer = new PrintWriter(socket.getOutputStream()); + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(socket.getInputStream()))) { + writer.write(getDeviceMsg(devices)); + writer.flush(); + // read query result, one line per requested device or ERROR followed by error message + while (true) { + final String line = bufferedReader.readLine(); + if (line == null) { + break; + } + if (line.equals(ERROR_STRING)) { + final String errorMsg = bufferedReader.lines().collect(Collectors.joining("\n")).strip(); + throw new DirectoryClientException(errorMsg); + } + result.add(parseDeviceInfo(line)); + } + } + } catch (IOException e) { + throw new DirectoryClientException("Nameserver error: ", e); + } + return result; + } + + private Device parseDeviceInfo(final String line) throws DirectoryClientException { + String[] tokens = line.split(" "); + if (tokens.length < 2) { + throw new DirectoryClientException("Malformed reply line: " + line); + } + if (tokens[1].equals(NOT_BOUND_LOCATION)) { + throw new DirectoryClientException("Requested device not bound: " + tokens[0]); + } + final ArrayList> servers = new ArrayList<>(); + for (int j = 2; j < tokens.length; j++) { + final HashMap server = new HashMap<>(); // NOPMD - necessary to allocate inside loop + servers.add(server); + final String[] servertokens = tokens[j].split("#"); + server.put("protocol", servertokens[0]); + int k = 1; + while (k + 3 < servertokens.length) { + if ("string".equals(servertokens[k + 1])) { + final int length = Integer.parseInt(servertokens[k + 2]); + final String value = URLDecoder.decode(servertokens[k + 3], Charset.defaultCharset()); + if (length == value.length()) { + server.put(servertokens[k], value); + } else { + throw new DirectoryClientException("Error parsing string: " + servertokens[k] + "(" + length + ") = " + value); + } + k += 4; + } else if ("int".equals(servertokens[k + 1]) || "long".equals(servertokens[k + 1])) { // NOPMD + server.put(servertokens[k], servertokens[k + 2]); + k += 3; + } else { + throw new DirectoryClientException("Error parsing argument: " + k + ": " + Arrays.toString(servertokens)); + } + } + } + return new Device(tokens[0], tokens[1], servers); + } + + public static class Device { + public final String name; + private final String deviceClass; + public final List> servers; + + public Device(final String name, final String deviceClass, final List> servers) { + this.name = name; + this.deviceClass = deviceClass; + this.servers = servers; + } + + @Override + public String toString() { + return "Device{name='" + name + '\'' + ", deviceClass='" + deviceClass + '\'' + ", servers=" + servers + '}'; + } + + public String getAddress() { + // N.B. here the '9' in 'rda3://9' is an indicator that the entry has 9 fields + // useful snippet for manual queries: + // echo -e "get-device-info\nGSCD025\n\n" | nc cmwpro00a.acc.gsi.de 5021 | sed -e "s%#%\n#%g" + return servers.stream().filter(s -> "rda3://9".equals(s.get("protocol"))).map(s -> s.get("Address:")).findFirst().orElseThrow(); + } + } + + public static class DirectoryClientException extends Exception { + private static final long serialVersionUID = -4452775634393421952L; + public DirectoryClientException(final String errorMsg) { + super(errorMsg); + } + public DirectoryClientException(final String errorMsg, final Exception cause) { + super(errorMsg, cause); + } + } +} diff --git a/client/src/main/java/io/opencmw/client/rest/RestDataSource.java b/client/src/main/java/io/opencmw/client/rest/RestDataSource.java new file mode 100644 index 00000000..c1815b25 --- /dev/null +++ b/client/src/main/java/io/opencmw/client/rest/RestDataSource.java @@ -0,0 +1,458 @@ + +package io.opencmw.client.rest; + +import static io.opencmw.OpenCmwProtocol.EMPTY_FRAME; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.LockSupport; +import java.util.concurrent.locks.ReentrantLock; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.spi.LoggingEventBuilder; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMQ.Socket; +import org.zeromq.ZMsg; + +import io.opencmw.MimeType; +import io.opencmw.client.DataSource; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.JsonSerialiser; + +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.sse.EventSource; +import okhttp3.sse.EventSourceListener; +import okhttp3.sse.EventSources; + +@SuppressWarnings({ "PMD.TooManyFields", "PMD.ExcessiveImports" }) +public class RestDataSource extends DataSource implements Runnable { + public static final Factory FACTORY = new Factory() { + @Override + public boolean matches(final String endpoint) { + return endpoint != null && !endpoint.isBlank() && endpoint.toLowerCase(Locale.UK).startsWith("http"); + } + + @Override + public Class getMatchingSerialiserType(final String endpoint) { + return JsonSerialiser.class; + } + + @Override + public DataSource newInstance(final ZContext context, final String endpoint, final Duration timeout, final String clientId) { + return new RestDataSource(context, endpoint, timeout, clientId); + } + }; + private static final Logger LOGGER = LoggerFactory.getLogger(RestDataSource.class); + private static final int WAIT_TIMEOUT_MILLIS = 1000; + private static final AtomicInteger REST_DATA_SOURCE_INSTANCE = new AtomicInteger(); + private static final int MAX_RETRIES = 3; + private static final AtomicLong PUBLICATION_COUNTER = new AtomicLong(); + protected static OkHttpClient okClient; + protected static EventSource.Factory eventSourceFactory; + protected final AtomicBoolean run = new AtomicBoolean(true); // NOPMD + protected final String uniqueID; + protected final byte[] uniqueIdBytes; + protected final String endpoint; + protected final Duration timeOut; + protected final String clientID; + protected int cancelLastCall; // needed for unit-testing only + protected final ZContext ctxCopy; + protected final Object newData = new Object(); // to notify event loop that new data has arrived + protected final Timer timer = new Timer(); + protected final List pendingCallbacks = Collections.synchronizedList(new ArrayList<>()); + protected final List completedCallbacks = Collections.synchronizedList(new ArrayList<>()); + protected final BlockingQueue requestQueue = new LinkedBlockingDeque<>(); + protected Map sseSource = new HashMap<>(); // NOPMD - only accessed from main thread + protected Socket internalSocket; // facing towards the internal REST client API + protected Socket externalSocket; // facing towards the DataSource manager + protected final TimerTask wakeupTask = new TimerTask() { + @Override + public void run() { + synchronized (newData) { + newData.notifyAll(); + } + } + }; + + protected RestDataSource(final ZContext ctx, final String endpoint) { + this(ctx, endpoint, Duration.ofMillis(0), RestDataSource.class.getName()); + } + + /** + * Constructor + * @param ctx ZeroMQ context to use + * @param endpoint Endpoint to subscribe to + * @param timeOut after which the request defaults to a time-out exception (no data) + * @param clientID subscription id to be able to process the notification updates. + */ + public RestDataSource(final ZContext ctx, final String endpoint, final Duration timeOut, final String clientID) { + super(endpoint); + synchronized (LOGGER) { // prevent race condition between multiple constructor invocations + // initialised only when needed, ie. when RestDataSource is actually instantiated + if (okClient == null) { + okClient = new OkHttpClient(); // NOPMD + eventSourceFactory = EventSources.createFactory(okClient); // NOPMD + } + } + + if (timeOut == null) { + throw new IllegalArgumentException("timeOut is null"); + } + this.ctxCopy = ctx == null ? new ZContext() : ctx; + this.endpoint = endpoint; + this.timeOut = timeOut; + this.clientID = clientID; + + uniqueID = clientID + "PID=" + ManagementFactory.getRuntimeMXBean().getName() + "-InstanceID=" + REST_DATA_SOURCE_INSTANCE.getAndIncrement(); + uniqueIdBytes = uniqueID.getBytes(ZMQ.CHARSET); + if (timeOut.toMillis() > 0) { + timer.scheduleAtFixedRate(wakeupTask, 0, timeOut.toMillis()); + } + + start(); // NOPMD - starts on initialisation + } + + /** + * Connect or reconnect to broker + */ + private void createPair() { + if (internalSocket != null) { + internalSocket.close(); + } + if (externalSocket != null) { + externalSocket.close(); + } + + internalSocket = ctxCopy.createSocket(SocketType.PAIR); + assert internalSocket != null : "internalSocket being initialised"; + if (!internalSocket.setHWM(0)) { + throw new IllegalStateException("could not set HWM on internalSocket"); + } + if (!internalSocket.setIdentity(uniqueIdBytes)) { + throw new IllegalStateException("could not set identity on internalSocket"); + } + if (!internalSocket.bind("inproc://" + uniqueID)) { + throw new IllegalStateException("could not bind internalSocket to: inproc://" + uniqueID); + } + + externalSocket = ctxCopy.createSocket(SocketType.PAIR); + assert externalSocket != null : "externalSocket being initialised"; + if (!externalSocket.setHWM(0)) { + throw new IllegalStateException("could not set HWM on externalSocket"); + } + if (!externalSocket.connect("inproc://" + uniqueID)) { + throw new IllegalStateException("could not bind externalSocket to: inproc://" + uniqueID); + } + + LOGGER.atTrace().addArgument(endpoint).log("(re-)connecting to REST endpoint: '{}'"); + } + /** + * Perform a get request on this endpoint. + * @param requestId request id which later allows to match the returned value to this query. + * This is the only mandatory parameter, all the following may be null. + * @param filterPattern extend the filters originally supplied to the endpoint e.g. "ctx=selector&channel=chanA" + * @param filters The serialised filters which will determine which data to update + * @param data The serialised data which can be used by the get call + * @param rbacToken byte array containing signed body hash-key and corresponding RBAC role + */ + @Override + public void get(final String requestId, final String filterPattern, final byte[] filters, final byte[] data, final byte[] rbacToken) { + enqueueRequest(requestId); //TODO: refactor interface + } + + /** + * Perform a set request on this endpoint using additional filters + * @param requestId request id which later allows to match the returned value to this query. + * This is the only mandatory parameter, all the following may be null. + * @param filterPattern extend the filters originally supplied to the endpoint e.g. "ctx=selector&channel=chanA" + * @param filters The serialised filters which will determine which data to update + * @param data The serialised data which can be used by the get call + * @param rbacToken byte array containing signed body hash-key and corresponding RBAC role + */ + @Override + public void set(final String requestId, final String filterPattern, final byte[] filters, final byte[] data, final byte[] rbacToken) { + throw new UnsupportedOperationException("set not (yet) implemented"); + } + + public void enqueueRequest(final String hashKey) { + if (!requestQueue.offer(hashKey)) { + throw new IllegalStateException("could not add hashKey " + hashKey + " to request queue of endpoint " + endpoint); + } + synchronized (newData) { + newData.notifyAll(); + } + } + + @Override + public void subscribe(final String reqId, final String endpoint, final byte[] rbacToken) { + final Request request = new Request.Builder().url(endpoint).build(); + sseSource.put(reqId, eventSourceFactory.newEventSource(request, new EventSourceListener() { + @Override + public void onEvent(final @NotNull EventSource eventSource, final String id, final String type, final @NotNull String data) { + final String pubKey = clientID + "#" + PUBLICATION_COUNTER.getAndIncrement(); + getRequest(pubKey, endpoint, MimeType.TEXT); // poll actual endpoint + } + })); + } + + @Override + public void unsubscribe(final String reqId) { + final EventSource source = sseSource.remove(reqId); + if (source != null) { + source.cancel(); + } + } + + public ZContext getCtx() { + return ctxCopy; + } + + @Override + public ZMQ.Socket getSocket() { + return externalSocket; + } + + @Override + protected Factory getFactory() { + return FACTORY; + } + + /** + * Gets called whenever data is available on the DataSoure's socket. + * Should then try to receive data and return any results back to the calling event loop. + * @return null if there is no more data available, a Zero length Zmsg if there was data which was only used internally + * or a ZMsg with [reqId, endpoint, byte[] data, [byte[] optional RBAC token]] + */ + @Override + public ZMsg getMessage() { + return ZMsg.recvMsg(externalSocket, false); + } + + @Override + public long housekeeping() { + synchronized (newData) { + ArrayList temp = new ArrayList<>(pendingCallbacks); + for (RestCallBack callBack : temp) { + callBack.checkTimeOut(); + } + + try { + while (!requestQueue.isEmpty()) { + final String hash = requestQueue.take(); + if (LOGGER.isTraceEnabled()) { + LOGGER.atTrace().addArgument(hash).log("external request with hashKey = '{}'"); + } + getRequest(hash, endpoint, MimeType.TEXT); + } + } catch (InterruptedException e) { // NOSONAR NOPMD + LOGGER.atError().setCause(e).addArgument(endpoint).log("error in retrieving requestQueue items for endpoint: {}"); + } + } + return System.currentTimeMillis() + timeOut.toMillis(); + } + + public void run() { // NOPMD NOSONAR - complexity + run.set(true); + try { + while (run.get() && !Thread.interrupted()) { + synchronized (newData) { + if (completedCallbacks.isEmpty() && requestQueue.isEmpty()) { + // nothing to do, wait for signals + final long waitMax; + if (timeOut.toMillis() <= 0) { + waitMax = TimeUnit.MILLISECONDS.toMillis(WAIT_TIMEOUT_MILLIS); + } else { + waitMax = timeOut.toMillis(); + } + // N.B. is automatically updated in case of time-out and/or new arriving data/exceptions + newData.wait(waitMax); + } + + for (RestCallBack callBack : completedCallbacks) { + // notify data + + final byte[] header; + final byte[] data; + if (callBack.response == null) { + // exception branch + header = EMPTY_FRAME; + data = EMPTY_FRAME; + } else { + header = callBack.response.headers().toString().getBytes(StandardCharsets.UTF_8); + data = callBack.response.peekBody(Long.MAX_VALUE).bytes(); + callBack.response.close(); + } + final byte[] exception = callBack.exception == null ? EMPTY_FRAME : callBack.exception.getMessage().getBytes(StandardCharsets.UTF_8); + + final ZMsg msg = new ZMsg(); // NOPMD - instantiation in loop + msg.add(callBack.hashKey); + msg.add(callBack.endPointName); + msg.add(header); + msg.add(data); + msg.add(exception); + + if (!msg.send(internalSocket)) { + throw new IllegalStateException("internalSocket could not send message - error code: " + internalSocket.errno()); + } + } + completedCallbacks.clear(); + + housekeeping(); + } + } + } catch (final Exception e) { // NOPMD NOSONAR -- terminate normally beyond this point + LOGGER.atError().setCause(e).log("data acquisition loop abnormally terminated"); + } finally { + externalSocket.close(); + internalSocket.close(); + } + LOGGER.atTrace().addArgument(uniqueID).addArgument(run.get()).log("stop poller thread for uniqueID={} - run={}"); + } + + public void start() { + createPair(); + new Thread(this).start(); // NOPMD + } + + public void stop() { + for (final String reqId : sseSource.keySet()) { + unsubscribe(reqId); + } + run.set(false); + } + + protected void getRequest(final String hashKey, final String path, final MimeType mimeType) { + Request request = new Request.Builder().url(path).get().addHeader("Accept", mimeType.toString()).build(); + if (LOGGER.isTraceEnabled()) { + LOGGER.atTrace().addArgument(endpoint).addArgument(path).addArgument(request).log("new request for {} - {} : request{}"); + } + final RestCallBack callBack = new RestCallBack(hashKey, path, mimeType); + pendingCallbacks.add(callBack); + final Call call = okClient.newCall(request); + call.enqueue(callBack); + if (cancelLastCall > 0) { + call.cancel(); // needed only for unit-testing + cancelLastCall--; + } + } + + public class RestCallBack implements Callback { + private final String hashKey; + private final String endPointName; + private final MimeType mimeType; + private final long requestTimeStamp = System.currentTimeMillis(); + private boolean active = true; + private final AtomicInteger retryCount = new AtomicInteger(); + private final Lock lock = new ReentrantLock(); + private Response response; + private Exception exception; + + public RestCallBack(final String hashKey, final String endPointName, final MimeType mimeType) { + this.hashKey = hashKey; + this.endPointName = endPointName; + this.mimeType = mimeType; + } + + @Override + public String toString() { + return "RestCallBack{hashKey='" + hashKey + '\'' + ", endPointName='" + endPointName + '\'' + ", requestTimeStamp=" + requestTimeStamp + ", active=" + active + ", retryCount=" + retryCount + ", result=" + response + ", exception=" + exception + '}'; + } + + public void checkTimeOut() { + if (!active || timeOut.toMillis() <= 0) { + return; + } + final long now = System.currentTimeMillis(); + if (requestTimeStamp + timeOut.toMillis() < now) { + // mark failed and notify + lock.lock(); + exception = new TimeoutException("ts=" + now + " - time-out of REST request for endpoint: " + endpoint); + notifyResult(); + lock.unlock(); + } + } + + @Override + public void onFailure(@NotNull final Call call, @NotNull final IOException e) { + if (!active) { + return; + } + + if (retryCount.incrementAndGet() <= MAX_RETRIES) { + lock.lock(); + exception = e; + lock.unlock(); + final LoggingEventBuilder logger = LOGGER.atWarn(); + if (LOGGER.isTraceEnabled()) { + logger.setCause(e); + } + logger.addArgument(retryCount.get()).addArgument(MAX_RETRIES).addArgument(endpoint).log("retry {} of {}: could not connect/receive from endpoint {}"); + // TODO: add more sophisticated exponential back-off + LockSupport.parkNanos(timeOut.toMillis() * (1L << (2 * (retryCount.get() - 1)))); + Request request = new Request.Builder().url(endPointName).get().addHeader("Accept", mimeType.toString()).build(); + final Call repeatedCall = okClient.newCall(request); + repeatedCall.enqueue(this); + if (cancelLastCall > 0) { + repeatedCall.cancel(); // needed only for unit-testing + cancelLastCall--; + } + return; + } + LOGGER.atWarn().setCause(e).addArgument(MAX_RETRIES).addArgument(endpoint).log("failed after {} connect/receive retries - abort"); + lock.lock(); + exception = e; + notifyResult(); + lock.unlock(); + LOGGER.atWarn().addArgument(e.getLocalizedMessage()).log("RestCallBack-Failure: '{}'"); + } + + @Override + public void onResponse(@NotNull final Call call, @NotNull final Response response) { + if (!active) { + return; + } + lock.lock(); + this.response = response; + notifyResult(); + lock.unlock(); + if (LOGGER.isTraceEnabled()) { + LOGGER.atTrace().addArgument(response).log("RestCallBack: '{}'"); + } + } + + private void notifyResult() { + synchronized (newData) { + active = false; + pendingCallbacks.remove(this); + completedCallbacks.add(this); + newData.notifyAll(); + } + } + } +} diff --git a/client/src/main/java/module-info.java b/client/src/main/java/module-info.java new file mode 100644 index 00000000..5e2e5b1e --- /dev/null +++ b/client/src/main/java/module-info.java @@ -0,0 +1,19 @@ +module io.opencmw.client { + requires java.management; + requires io.opencmw; + requires io.opencmw.serialiser; + requires org.slf4j; + requires jeromq; + requires disruptor; + requires org.jetbrains.annotations; + requires okhttp3; + requires okhttp3.sse; + requires kotlin.stdlib; + requires it.unimi.dsi.fastutil; + + exports io.opencmw.client.cmwlight; + exports io.opencmw.client.rest; + exports io.opencmw.client; + + opens io.opencmw.client to io.opencmw.serialiser; +} \ No newline at end of file diff --git a/client/src/test/java/io/opencmw/client/DataSourceExample.java b/client/src/test/java/io/opencmw/client/DataSourceExample.java new file mode 100644 index 00000000..90e4fd13 --- /dev/null +++ b/client/src/test/java/io/opencmw/client/DataSourceExample.java @@ -0,0 +1,45 @@ +package io.opencmw.client; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.EventStore; +import io.opencmw.client.cmwlight.CmwLightExample; +import io.opencmw.client.cmwlight.DirectoryLightClient; +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; + +public class DataSourceExample { + private final static Logger LOGGER = LoggerFactory.getLogger(DataSourceExample.class); + private final static String DEV_NAME = "GSCD002"; + private final static String DEV2_NAME = "GSCD001"; + private final static String PROP = "AcquisitionDAQ"; + private final static String SELECTOR = "FAIR.SELECTOR.ALL"; + + public static void main(String[] args) throws DirectoryLightClient.DirectoryClientException { + // create and start a simple event store which just prints everything written to it to stdout + final EventStore eventStore = EventStore.getFactory().setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + eventStore.register((e, s, last) -> { + System.out.println(e); + System.out.println(e.payload.get()); + }); + eventStore.start(); + // create a data source publisher and add a subscription + final DataSourcePublisher dataSourcePublisher = new DataSourcePublisher(null, eventStore); + if (args.length == 0) { + LOGGER.atError().log("no directory server supplied"); + return; + } + LOGGER.atInfo().addArgument(args[0]).log("directory server: {}"); + final DirectoryLightClient directoryLightClient = new DirectoryLightClient(args[0]); + final List devInfo = directoryLightClient.getDeviceInfo(List.of(DEV_NAME, DEV2_NAME)); + final String address = devInfo.get(0).getAddress().replace("tcp://", "rda3://"); + final String address2 = devInfo.get(1).getAddress().replace("tcp://", "rda3://"); + // run the publisher's main loop + new Thread(dataSourcePublisher).start(); + dataSourcePublisher.subscribe(address + '/' + DEV_NAME + '/' + PROP + "?ctx=" + SELECTOR + '&' + "acquisitionModeFilter=int:0" + '&' + "channelNameFilter=GS11MU2:Current_1@10Hz", CmwLightExample.AcquisitionDAQ.class); + dataSourcePublisher.subscribe(address2 + '/' + DEV2_NAME + '/' + PROP + "?ctx=FAIR.SELECTOR.ALL" + '&' + "acquisitionModeFilter=int:4&channelNameFilter=GS02P:SumY:Triggered@25MHz", CmwLightExample.AcquisitionDAQ.class); + } +} diff --git a/client/src/test/java/io/opencmw/client/DataSourcePublisherTest.java b/client/src/test/java/io/opencmw/client/DataSourcePublisherTest.java new file mode 100644 index 00000000..0415d43b --- /dev/null +++ b/client/src/test/java/io/opencmw/client/DataSourcePublisherTest.java @@ -0,0 +1,299 @@ +package io.opencmw.client; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.EventStore; +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; + +class DataSourcePublisherTest { + private static final AtomicReference testObject = new AtomicReference<>(); + private final Integer requestBody = 10; + private final Map requestFilter = new HashMap<>(); + + private DataSourcePublisher.ThePromisedFuture getTestFuture() { + return new DataSourcePublisher.ThePromisedFuture<>("endPoint", requestFilter, requestBody, Float.class, DataSourceFilter.ReplyType.GET, "TestClientID"); + } + + private static class TestDataSource extends DataSource { + public static final Factory FACTORY = new Factory() { + @Override + public boolean matches(final String endpoint) { + return endpoint.startsWith("test://"); + } + + @Override + public Class getMatchingSerialiserType(final String endpoint) { + return BinarySerialiser.class; + } + + @Override + public DataSource newInstance(final ZContext context, final String endpoint, final Duration timeout, final String clientId) { + return new TestDataSource(context, endpoint, timeout, clientId); + } + }; + private final static String INPROC = "inproc://testDataSource"; + private final ZContext context; + private final ZMQ.Socket socket; + private ZMQ.Socket internalSocket; + private long nextHousekeeping = 0; + private final IoClassSerialiser ioClassSerialiser = new IoClassSerialiser(new FastByteBuffer(2000)); + private final Map subscriptions = new HashMap<>(); + private final Map requests = new HashMap<>(); + private long nextNotification = 0L; + + public TestDataSource(final ZContext context, final String endpoint, final Duration timeOut, final String clientId) { + super(endpoint); + this.context = context; + this.socket = context.createSocket(SocketType.DEALER); + this.socket.bind(INPROC); + } + + @Override + public long housekeeping() { + final long currentTime = System.currentTimeMillis(); + if (currentTime > nextHousekeeping) { + if (currentTime > nextNotification) { + subscriptions.forEach((subscriptionId, endpoint) -> { + if (internalSocket == null) { + internalSocket = context.createSocket(SocketType.DEALER); + internalSocket.connect(INPROC); + } + final ZMsg msg = new ZMsg(); + msg.add(subscriptionId); + msg.add(endpoint); + msg.add(new byte[0]); // header + ioClassSerialiser.getDataBuffer().reset(); + ioClassSerialiser.serialiseObject(testObject.get()); + // todo: the serialiser does not report the correct position after serialisation + msg.add(Arrays.copyOfRange(ioClassSerialiser.getDataBuffer().elements(), 0, ioClassSerialiser.getDataBuffer().position() + 4)); + msg.add(new byte[0]); // exception + msg.send(internalSocket); + }); + nextNotification = currentTime + 3000; + } + requests.forEach((requestId, endpoint) -> { + if (internalSocket == null) { + internalSocket = context.createSocket(SocketType.DEALER); + internalSocket.connect(INPROC); + } + final ZMsg msg = new ZMsg(); + msg.add(requestId); + msg.add(endpoint); + msg.add(new byte[0]); // header + ioClassSerialiser.getDataBuffer().reset(); + ioClassSerialiser.serialiseObject(testObject.get()); + // todo: the serialiser does not report the correct position after serialisation + msg.add(Arrays.copyOfRange(ioClassSerialiser.getDataBuffer().elements(), 0, ioClassSerialiser.getDataBuffer().position() + 4)); + msg.add(new byte[0]); // exception + msg.send(internalSocket); + }); + requests.clear(); + nextHousekeeping = currentTime + 200; + } + return nextHousekeeping; + } + + @Override + public void get(final String requestId, final String endpoint, final byte[] filters, final byte[] data, final byte[] rbacToken) { + requests.put(requestId, endpoint); + } + + @Override + public void set(final String requestId, final String endpoint, final byte[] filters, final byte[] data, final byte[] rbacToken) { + throw new UnsupportedOperationException("cannot perform set"); + } + + @Override + public ZMQ.Socket getSocket() { + return socket; + } + + @Override + protected Factory getFactory() { + return FACTORY; + } + + @Override + public ZMsg getMessage() { + return ZMsg.recvMsg(socket, ZMQ.DONTWAIT); + } + + @Override + public void subscribe(final String reqId, final String endpoint, final byte[] rbacToken) { + subscriptions.put(reqId, endpoint); + } + + @Override + public void unsubscribe(final String reqId) { + subscriptions.remove(reqId); + } + } + + public static class TestObject { + private final String foo; + private final double bar; + + public TestObject(final String foo, final double bar) { + this.foo = foo; + this.bar = bar; + } + + public TestObject() { + this.foo = ""; + this.bar = Double.NaN; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (!(o instanceof TestObject)) + return false; + final TestObject that = (TestObject) o; + return bar == that.bar && Objects.equals(foo, that.foo); + } + + @Override + public int hashCode() { + return Objects.hash(foo, bar); + } + + @Override + public String + toString() { + return "TestObject{" + + "foo='" + foo + '\'' + ", bar=" + bar + '}'; + } + } + + @Test + void testSubscribe() { + final AtomicBoolean eventReceived = new AtomicBoolean(false); + final TestObject referenceObject = new TestObject("foo", 1.337); + testObject.set(referenceObject); + + DataSource.register(TestDataSource.FACTORY); + + final EventStore eventStore = EventStore.getFactory().setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + eventStore.register((event, sequence, endOfBatch) -> { + assertEquals(testObject.get(), event.payload.get(TestObject.class)); + eventReceived.set(true); + }); + + final DataSourcePublisher dataSourcePublisher = new DataSourcePublisher(null, eventStore); + + eventStore.start(); + new Thread(dataSourcePublisher).start(); + + dataSourcePublisher.subscribe("test://foobar/testdev/prop?ctx=FAIR.SELECTOR.ALL&filter=foobar", TestObject.class); + + Awaitility.waitAtMost(Duration.ofSeconds(1)).until(eventReceived::get); + } + + @Test + void testGet() throws InterruptedException, ExecutionException, TimeoutException { + final TestObject referenceObject = new TestObject("foo", 1.337); + testObject.set(referenceObject); + + DataSource.register(TestDataSource.FACTORY); + + final EventStore eventStore = EventStore.getFactory().setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + final DataSourcePublisher dataSourcePublisher = new DataSourcePublisher(null, eventStore); + + eventStore.start(); + new Thread(dataSourcePublisher).start(); + + final Future future = dataSourcePublisher.get("test://foobar/testdev/prop?ctx=FAIR.SELECTOR.ALL&filter=foobar", TestObject.class); + + final TestObject result = future.get(1000, TimeUnit.MILLISECONDS); + assertEquals(referenceObject, result); + } + + @Test + void testGetTimeout() { + testObject.set(null); // makes the test event source not answer the get request + + DataSource.register(TestDataSource.FACTORY); + + final EventStore eventStore = EventStore.getFactory().setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + final DataSourcePublisher dataSourcePublisher = new DataSourcePublisher(null, eventStore); + + eventStore.start(); + new Thread(dataSourcePublisher).start(); + + final Future future = dataSourcePublisher.get("test://foobar/testdev/prop?ctx=FAIR.SELECTOR.ALL&filter=foobar", TestObject.class); + + assertThrows(TimeoutException.class, () -> future.get(1, TimeUnit.MILLISECONDS)); + } + + @Test + void testFuture() throws InterruptedException, ExecutionException { + final Float replyObject = (float) Math.PI; + + { + final DataSourcePublisher.ThePromisedFuture future = getTestFuture(); + + assertNotNull(future); + assertEquals("endPoint", future.getEndpoint()); + assertEquals(requestFilter, future.getRequestFilter()); + assertEquals(requestBody, future.getRequestBody()); + assertEquals(Float.class, future.getRequestedDomainObjType()); + assertEquals(DataSourceFilter.ReplyType.GET, future.getReplyType()); + assertEquals("TestClientID", future.getInternalRequestID()); + + future.setReply(replyObject); + assertEquals(replyObject, future.get()); + assertFalse(future.cancel(true)); + } + + { + // delayed reply + final DataSourcePublisher.ThePromisedFuture future = getTestFuture(); + new Timer().schedule(new TimerTask() { + @Override + public void run() { + future.setReply(replyObject); + } + }, 300); + Awaitility.waitAtMost(Duration.ofSeconds(1)).until(() -> replyObject.equals(future.get())); + } + + { + // cancelled reply + final DataSourcePublisher.ThePromisedFuture future = getTestFuture(); + assertFalse(future.isCancelled()); + future.cancel(true); + assertTrue(future.isCancelled()); + assertThrows(CancellationException.class, () -> future.get(1, TimeUnit.SECONDS)); + } + } +} diff --git a/client/src/test/java/io/opencmw/client/EndpointTest.java b/client/src/test/java/io/opencmw/client/EndpointTest.java new file mode 100644 index 00000000..8a6ac542 --- /dev/null +++ b/client/src/test/java/io/opencmw/client/EndpointTest.java @@ -0,0 +1,21 @@ +package io.opencmw.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.Test; + +public class EndpointTest { + @Test + void testEndpointParsing() { + final Endpoint ep = new Endpoint("rda3://server:port/device/property?ctx=test.sel:t=100&filter=asdf&amount=int:1"); + assertEquals("rda3://", ep.getProtocol()); + assertEquals("rda3://server:port", ep.getAddress()); + assertEquals("device", ep.getDevice()); + assertEquals("property", ep.getProperty()); + assertEquals("test.sel:t=100", ep.getSelector()); + assertEquals(Map.of("filter", "asdf", "amount", 1), ep.getFilters()); + assertEquals("rda3://server:port/device/property?ctx=test.sel:t=101:id=1&filter=asdf&amount=int:1", ep.getEndpointForContext("test.sel:t=101:id=1")); + } +} diff --git a/client/src/test/java/io/opencmw/client/cmwlight/CmwLightDataSourceTest.java b/client/src/test/java/io/opencmw/client/cmwlight/CmwLightDataSourceTest.java new file mode 100644 index 00000000..5bebd870 --- /dev/null +++ b/client/src/test/java/io/opencmw/client/cmwlight/CmwLightDataSourceTest.java @@ -0,0 +1,117 @@ +package io.opencmw.client.cmwlight; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.Charset; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.locks.LockSupport; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.zeromq.*; + +import io.opencmw.client.Endpoint; + +class CmwLightDataSourceTest { + @Test + void testCmwLightSubscription() throws CmwLightProtocol.RdaLightException { + // setup zero mq socket to mock cmw server + ZContext context = new ZContext(1); + ZMQ.Socket socket = context.createSocket(SocketType.DEALER); + socket.bind("tcp://localhost:7777"); + + final CmwLightDataSource client = new CmwLightDataSource(context, "rda3://localhost:7777/testdevice/testprop?ctx=test.selector&nFilter=int:1", "testClientId"); + + client.connect(); + client.housekeeping(); + + // check connection request was received + final CmwLightMessage connectMsg = CmwLightProtocol.parseMsg(ZMsg.recvMsg(socket)); + assertEquals(CmwLightProtocol.MessageType.CLIENT_CONNECT, connectMsg.messageType); + assertEquals(CmwLightProtocol.VERSION, connectMsg.version); + client.housekeeping(); // allow the subscription to be sent out + + // send connection ack + CmwLightProtocol.sendMsg(socket, CmwLightMessage.connectAck("1.3.7")); + CmwLightProtocol.sendMsg(socket, CmwLightMessage.SERVER_HB); + client.getMessage(); // Make client receive ack and update connection status + client.housekeeping(); // allow the subscription to be sent out + + // assert that the client has connected + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> { + client.getMessage(); // Make client receive ack and update connection status + client.housekeeping(); // allow the subscription to be sent out + return client.connectionState.get().equals(CmwLightDataSource.ConnectionState.CONNECTED); + }); + + // request subscription + final String reqId = "testId"; + final String endpoint = "rda3://localhost:7777/testdevice/testprop?ctx=FAIR.SELECTOR.ALL&nFilter=int:1"; + client.subscribe(reqId, endpoint, null); + + final CmwLightMessage subMsg = getNextNonHeartbeatMsg(socket, client, false); + assertEquals(CmwLightProtocol.MessageType.CLIENT_REQ, subMsg.messageType); + assertEquals(CmwLightProtocol.RequestType.SUBSCRIBE, subMsg.requestType); + assertEquals(Map.of("nFilter", 1), subMsg.requestContext.filters); + + // acknowledge subscription + final long sourceId = 1337L; + CmwLightProtocol.sendMsg(socket, CmwLightMessage.subscribeReply(subMsg.sessionId, subMsg.id, subMsg.deviceName, subMsg.propertyName, Map.of(CmwLightProtocol.FieldName.SOURCE_ID_TAG.value(), sourceId))); + + // assert that the subscription was established + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> { + client.getMessage(); // Make client receive ack and update connection status + client.housekeeping(); // allow the subscription to be sent out + return client.replyIdMap.containsKey(sourceId); + }); + + // send 10 updates + for (int i = 0; i < 10; i++) { + final String cycleName = "FAIR.SELECTOR.C=" + (i + 1); + CmwLightProtocol.sendMsg(socket, CmwLightMessage.notificationReply(subMsg.sessionId, sourceId, "", "", new ZFrame("data"), i, + new CmwLightMessage.DataContext(cycleName, 123456789, 123456788, null), CmwLightProtocol.UpdateType.NORMAL)); + + // assert that the subscription update was received + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> { + final ZMsg reply = client.getMessage(); // Make client receive ack and update connection status + client.housekeeping(); // allow the subscription to be sent out + + return reply.size() == 5 && reply.pollFirst().getString(Charset.defaultCharset()).equals("testId") + && Objects.requireNonNull(reply.pollFirst()).getString(Charset.defaultCharset()).equals(new Endpoint(endpoint).getEndpointForContext(cycleName)) + && Objects.requireNonNull(reply.pollFirst()).getData().length == 0 + && Objects.requireNonNull(reply.pollFirst()).getString(Charset.defaultCharset()).equals("data") + && Objects.requireNonNull(reply.pollFirst()).getData().length == 0; + }); + } + } + + /* + / get next message sent from client to server ignoring heartbeats, periodically send heartbeat and perform housekeeping + */ + private CmwLightMessage getNextNonHeartbeatMsg(final ZMQ.Socket socket, final CmwLightDataSource client, boolean debug) throws CmwLightProtocol.RdaLightException { + int i = 0; + while (true) { + final ZMsg msg = ZMsg.recvMsg(socket, false); + final CmwLightMessage result = msg == null ? null : CmwLightProtocol.parseMsg(msg); + if (debug) { + if (result == null) { + System.out.print('.'); + } else { + System.out.println(result); + } + } + if (result != null && result.messageType != CmwLightProtocol.MessageType.CLIENT_HB) { + return result; + } + if (i % 10 == 0) { // send server heartbeat every second + CmwLightProtocol.sendMsg(socket, CmwLightMessage.SERVER_HB); + } + client.housekeeping(); + client.getMessage(); + LockSupport.parkNanos(100000); + i++; + } + } +} diff --git a/client/src/test/java/io/opencmw/client/cmwlight/CmwLightExample.java b/client/src/test/java/io/opencmw/client/cmwlight/CmwLightExample.java new file mode 100644 index 00000000..faf5aa15 --- /dev/null +++ b/client/src/test/java/io/opencmw/client/cmwlight/CmwLightExample.java @@ -0,0 +1,119 @@ +package io.opencmw.client.cmwlight; + +import java.util.Arrays; +import java.util.Collections; + +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; + +public class CmwLightExample { // NOPMD is not a utility class but a sample + private final static String CMW_NAMESERVER = "cmwpro00a.acc.gsi.de:5021"; + private final static String DEVICE = "GSCD002"; + // private final static String PROPERTY = "SnoopTriggerEvents"; + private final static String PROPERTY = "AcquisitionDAQ"; + private final static String SELECTOR = "FAIR.SELECTOR.ALL"; + + public static void main(String[] args) throws DirectoryLightClient.DirectoryClientException { + subscribeAcqFromDigitizer(); + } + + public static void subscribeAcqFromDigitizer() throws DirectoryLightClient.DirectoryClientException { + final DirectoryLightClient directoryClient = new DirectoryLightClient(CMW_NAMESERVER); + DirectoryLightClient.Device device = directoryClient.getDeviceInfo(Collections.singletonList(DEVICE)).get(0); + System.out.println(device); + final String address = device.servers.stream().findFirst().orElseThrow().get("Address:"); + System.out.println("connect client to " + address); + final CmwLightDataSource client = new CmwLightDataSource(new ZContext(1), address, "testclient"); + final ZMQ.Poller poller = client.getContext().createPoller(1); + poller.register(client.getSocket(), ZMQ.Poller.POLLIN); + client.connect(); + + System.out.println("starting subscription"); + // 4 = Triggered Acquisition Mode; 0 = Continuous Acquisition mode + String filtersString = "acquisitionModeFilter=int:0&channelNameFilter=GS11MU2:Current_1@10Hz"; + String filters2String = "acquisitionModeFilter=int:0&channelNameFilter=GS11MU2:Voltage_1@10Hz"; + client.subscribe("r1", "rda3://" + DEVICE + '/' + DEVICE + '/' + PROPERTY + "?ctx=" + SELECTOR + "&" + filtersString, null); + client.subscribe("r1", "rda3://" + DEVICE + '/' + DEVICE + '/' + PROPERTY + "?ctx=" + SELECTOR + "&" + filters2String, null); + client.subscriptions.forEach((id, c) -> System.out.println(id + " -> " + c)); + + int i = 0; + while (i < 45) { + client.housekeeping(); + poller.poll(); + final CmwLightMessage result = client.receiveData(); + if (result != null && result.requestType == CmwLightProtocol.RequestType.NOTIFICATION_DATA) { + System.out.println(result); + final byte[] bytes = result.bodyData.getData(); + final IoClassSerialiser classSerialiser = new IoClassSerialiser(FastByteBuffer.wrap(bytes), CmwLightSerialiser.class); + final AcquisitionDAQ acq = classSerialiser.deserialiseObject(AcquisitionDAQ.class); + System.out.println("body: " + acq); + } else { + if (result != null) + System.out.println(result); + } + if (i == 15) { + client.subscriptions.forEach((id, c) -> System.out.println(id + " -> " + c)); + System.out.println("unsubscribe"); + client.unsubscribe("r1"); + client.unsubscribe("r2"); + } + i++; + } + } + + public static class AcquisitionDAQ { + public String refTriggerName; + public long refTriggerStamp; + public float[] channelTimeSinceRefTrigger; + public float channelUserDelay; + public float channelActualDelay; + public String channelName; + public float[] channelValue; + public float[] channelError; + public String channelUnit; + public int status; + public float channelRangeMin; + public float channelRangeMax; + public float temperature; + public int processIndex; + public int sequenceIndex; + public int chainIndex; + public int eventNumber; + public int timingGroupId; + public long acquisitionStamp; + public long eventStamp; + public long processStartStamp; + public long sequenceStartStamp; + public long chainStartStamp; + + @Override + public String toString() { + return "AcquisitionDAQ{" + + "refTriggerName='" + refTriggerName + '\'' + ", refTriggerStamp=" + refTriggerStamp + ", channelTimeSinceRefTrigger(n=" + channelTimeSinceRefTrigger.length + ")=" + Arrays.toString(Arrays.copyOfRange(channelTimeSinceRefTrigger, 0, 3)) + ", channelUserDelay=" + channelUserDelay + ", channelActualDelay=" + channelActualDelay + ", channelName='" + channelName + '\'' + ", channelValue(n=" + channelValue.length + ")=" + Arrays.toString(Arrays.copyOfRange(channelValue, 0, 3)) + ", channelError(n=" + channelError.length + ")=" + Arrays.toString(Arrays.copyOfRange(channelError, 0, 3)) + ", channelUnit='" + channelUnit + '\'' + ", status=" + status + ", channelRangeMin=" + channelRangeMin + ", channelRangeMax=" + channelRangeMax + ", temperature=" + temperature + ", processIndex=" + processIndex + ", sequenceIndex=" + sequenceIndex + ", chainIndex=" + chainIndex + ", eventNumber=" + eventNumber + ", timingGroupId=" + timingGroupId + ", acquisitionStamp=" + acquisitionStamp + ", eventStamp=" + eventStamp + ", processStartStamp=" + processStartStamp + ", sequenceStartStamp=" + sequenceStartStamp + ", chainStartStamp=" + chainStartStamp + '}'; + } + } + + public static class SnoopAcquisition { + public String TriggerEventName; + public long acquisitionStamp; + public int chainIndex; + public long chainStartStamp; + public int eventNumber; + public long eventStamp; + public int processIndex; + public long processStartStamp; + public int sequenceIndex; + public long sequenceStartStamp; + public int timingGroupID; + + @Override + public String toString() { + return "SnoopAcquisition{" + + "TriggerEventName='" + TriggerEventName + '\'' + ", acquisitionStamp=" + acquisitionStamp + ", chainIndex=" + chainIndex + ", chainStartStamp=" + chainStartStamp + ", eventNumber=" + eventNumber + ", eventStamp=" + eventStamp + ", processIndex=" + processIndex + ", processStartStamp=" + processStartStamp + ", sequenceIndex=" + sequenceIndex + ", sequenceStartStamp=" + sequenceStartStamp + ", timingGroupID=" + timingGroupID + '}'; + } + } +} diff --git a/client/src/test/java/io/opencmw/client/cmwlight/CmwLightProtocolTest.java b/client/src/test/java/io/opencmw/client/cmwlight/CmwLightProtocolTest.java new file mode 100644 index 00000000..22dffdcc --- /dev/null +++ b/client/src/test/java/io/opencmw/client/cmwlight/CmwLightProtocolTest.java @@ -0,0 +1,210 @@ +package io.opencmw.client.cmwlight; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.zeromq.ZFrame; +import org.zeromq.ZMsg; + +/** + * Test serialisation and deserialisation of cmw protocol messages. + */ +class CmwLightProtocolTest { + @Test + void testConnectRequest() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.connectRequest("testsession", + 1337L, + "testdev", + "testprop"); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testEventRequest() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.eventRequest("testsession", + 1337L, + "testdev", + "testprop"); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testEventReply() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.eventReply("testsession", + 1337L, + "testdev", + "testprop"); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + @Disabled // issues with the empty map in options in the cmw light serialiser + void testSessionConfirmReply() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.sessionConfirmReply("testsession", + 1337L, + "testdev", + "testprop", + Map.of(CmwLightProtocol.FieldName.SESSION_BODY_TAG.value(), Collections.emptyMap())); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + serialised.forEach(frame -> System.out.println(frame.getString(Charset.defaultCharset()))); + serialised.forEach(frame -> System.out.println(Arrays.toString(frame.getData()))); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testSessionGetReply() throws CmwLightProtocol.RdaLightException { + final ZFrame data = new ZFrame(new byte[] { 0, 1, 4, 7, 8 }); + final CmwLightMessage subscriptionMsg = CmwLightMessage.getReply("testsession", + 1338L, + "testdev", + "testprop", + data, + new CmwLightMessage.DataContext("testCycleName", 4242, 2323, null)); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testNotificationReply() throws CmwLightProtocol.RdaLightException { + final ZFrame data = new ZFrame(new byte[] { 0, 1, 4, 7, 8 }); + final CmwLightMessage subscriptionMsg = CmwLightMessage.notificationReply("testsession", + 1337L, + "testdev", + "testprop", + data, + 7, + new CmwLightMessage.DataContext("testCycleName", 4242, 2323, null), + CmwLightProtocol.UpdateType.IMMEDIATE_UPDATE); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testExceptionReply() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.exceptionReply( + "testsession", 1337L, "testdev", "testprop", + "test exception message", 314, 981, (byte) 3); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testNotificationExceptionReply() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.notificationExceptionReply( + "testsession", 1337L, "testdev", "testprop", + "test exception message", 314, 981, (byte) 3); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testSubscribeExceptionReply() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.subscribeExceptionReply( + "testsession", 1337L, "testdev", "testprop", + "test exception message", 314, 981, (byte) 3); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testSubscribeRequest() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.subscribeRequest( + "testsession", 1337L, "testdev", "testprop", + Map.of("b", 1337L), + new CmwLightMessage.RequestContext("testselector", Map.of("testfilter", 5L), null), + CmwLightProtocol.UpdateType.NORMAL); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + // System.out.println(serialised); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testUnsubscribeRequest() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.unsubscribeRequest("testsession", + 1337L, + "testdev", + "testprop", + Map.of("b", 1337L), + CmwLightProtocol.UpdateType.NORMAL); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testGetRequest() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage subscriptionMsg = CmwLightMessage.getRequest("testsession", + 1337L, + "testdev", + "testprop", + new CmwLightMessage.RequestContext("testselector", Map.of("testfilter", 5L), null)); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testSetRequest() throws CmwLightProtocol.RdaLightException { + final ZFrame data = new ZFrame(new byte[] { 0, 1, 4, 7, 8 }); + final CmwLightMessage subscriptionMsg = CmwLightMessage.setRequest("testsession", + 1337L, + "testdev", + "testprop", + data, + new CmwLightMessage.RequestContext("testselector", Map.of("testfilter", 5L), null)); + ZMsg serialised = CmwLightProtocol.serialiseMsg(subscriptionMsg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(subscriptionMsg, restored); + } + + @Test + void testHbServerMsg() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage msg = CmwLightMessage.SERVER_HB; + ZMsg serialised = CmwLightProtocol.serialiseMsg(msg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(msg, restored); + } + + @Test + void testHbClientMsg() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage msg = CmwLightMessage.CLIENT_HB; + ZMsg serialised = CmwLightProtocol.serialiseMsg(msg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(msg, restored); + } + + @Test + void testConnectMsg() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage msg = CmwLightMessage.connect("1.3.7"); + ZMsg serialised = CmwLightProtocol.serialiseMsg(msg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(msg, restored); + } + + @Test + void testConnectAckMsg() throws CmwLightProtocol.RdaLightException { + final CmwLightMessage msg = CmwLightMessage.connectAck("1.3.7"); + ZMsg serialised = CmwLightProtocol.serialiseMsg(msg); + final CmwLightMessage restored = CmwLightProtocol.parseMsg(serialised); + assertEquals(msg, restored); + } +} diff --git a/client/src/test/java/io/opencmw/client/rest/Event.java b/client/src/test/java/io/opencmw/client/rest/Event.java new file mode 100644 index 00000000..7a25c33d --- /dev/null +++ b/client/src/test/java/io/opencmw/client/rest/Event.java @@ -0,0 +1,41 @@ +package io.opencmw.client.rest; + +import java.util.Objects; + +public class Event { + private final String id; + private final String type; + private final String data; + + public Event(final String id, final String type, final String data) { + if (data == null) { + throw new IllegalArgumentException("data == null"); + } + this.id = id; + this.type = type; + this.data = data; + } + + @Override + public String toString() { + return "Event{id='" + id + "', type='" + type + "', data='" + data + "'}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof Event)) + return false; + Event other = (Event) o; + return Objects.equals(id, other.id) && Objects.equals(type, other.type) && data.equals(other.data); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(id); + result = 31 * result + Objects.hashCode(type); + result = 31 * result + data.hashCode(); + return result; + } +} diff --git a/client/src/test/java/io/opencmw/client/rest/EventSourceRecorder.java b/client/src/test/java/io/opencmw/client/rest/EventSourceRecorder.java new file mode 100644 index 00000000..65f055b0 --- /dev/null +++ b/client/src/test/java/io/opencmw/client/rest/EventSourceRecorder.java @@ -0,0 +1,142 @@ +package io.opencmw.client.rest; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; + +import org.jetbrains.annotations.NotNull; + +import okhttp3.Response; +import okhttp3.internal.platform.Platform; +import okhttp3.sse.EventSource; +import okhttp3.sse.EventSourceListener; + +class EventSourceRecorder extends EventSourceListener { + private final BlockingQueue events = new LinkedBlockingDeque<>(); + + @Override + public void onOpen(@NotNull EventSource eventSource, @NotNull Response response) { + Platform.get().log("[ES] onOpen", Platform.INFO, null); + events.add(new Open(eventSource, response)); + } + + @Override + public void onEvent(@NotNull EventSource eventSource, String id, String type, @NotNull String data) { + Platform.get().log("[ES] onEvent", Platform.INFO, null); + events.add(new Event(id, type, data)); + } + + @Override + public void onClosed(@NotNull EventSource eventSource) { + Platform.get().log("[ES] onClosed", Platform.INFO, null); + events.add(new Closed()); + } + + @Override + public void onFailure(@NotNull EventSource eventSource, Throwable t, Response response) { + Platform.get().log("[ES] onFailure", Platform.INFO, t); + events.add(new Failure(t, response)); + } + + private Object nextEvent() { + try { + Object event = events.poll(10, SECONDS); + if (event == null) { + throw new AssertionError("Timed out waiting for event."); + } + return event; + } catch (InterruptedException e) { + throw new AssertionError(e); + } + } + + public void assertExhausted() { + assertTrue(events.isEmpty()); + } + + public void assertEvent(String id, String type, String data) { + Object actual = nextEvent(); + assertEquals(new Event(id, type, data), actual); + } + + public EventSource assertOpen() { + Object event = nextEvent(); + if (!(event instanceof Open)) { + throw new AssertionError("Expected Open but was " + event); + } + return ((Open) event).eventSource; + } + + public void assertClose() { + Object event = nextEvent(); + if (!(event instanceof Closed)) { + throw new AssertionError("Expected Open but was " + event); + } + } + + public void assertFailure(String message) { + Object event = nextEvent(); + if (!(event instanceof Failure)) { + throw new AssertionError("Expected Failure but was " + event); + } + if (message != null) { + assertEquals(message, ((Failure) event).t.getMessage()); + } else { + assertNull(((Failure) event).t); + } + } + + static final class Open { + final EventSource eventSource; + final Response response; + + Open(EventSource eventSource, Response response) { + this.eventSource = eventSource; + this.response = response; + } + + @Override + public String toString() { + return "Open[" + response + ']'; + } + } + + static final class Failure { + final Throwable t; + final Response response; + final String responseBody; + + Failure(Throwable t, Response response) { + this.t = t; + this.response = response; + String responseBody = null; + if (response != null) { + try { + responseBody = Objects.requireNonNull(response.body()).string(); + } catch (IOException ignored) { + } + } + this.responseBody = responseBody; + } + + @Override + public String toString() { + if (response == null) { + return "Failure[" + t + "]"; + } + return "Failure[" + response + "]"; + } + } + + static final class Closed { + @Override + public String toString() { + return "Closed[]"; + } + } +} diff --git a/client/src/test/java/io/opencmw/client/rest/RestDataSourceTest.java b/client/src/test/java/io/opencmw/client/rest/RestDataSourceTest.java new file mode 100644 index 00000000..39693ae7 --- /dev/null +++ b/client/src/test/java/io/opencmw/client/rest/RestDataSourceTest.java @@ -0,0 +1,251 @@ +package io.opencmw.client.rest; + +import static java.util.Objects.requireNonNull; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMQException; +import org.zeromq.ZMsg; + +import io.opencmw.MimeType; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okhttp3.sse.EventSource; +import okhttp3.sse.EventSources; +import zmq.ZError; + +class RestDataSourceTest { + private static final Logger LOGGER = LoggerFactory.getLogger(RestDataSourceTest.class); + private static final String TEST_DATA = "Hello World!"; + private static final int DEFAULT_TIMEOUT_MILLIS = 1000; + private static final int DEFAULT_WAIT_MILLIS = 10; + private MockWebServer server; + private OkHttpClient client; + private EventSourceRecorder listener; + + @BeforeEach + void before() throws IOException { + this.server = new MockWebServer(); + server.setDispatcher(new CustomDispatcher()); + server.start(); + + client = new OkHttpClient(); + listener = new EventSourceRecorder(); + } + + @AfterEach + void after() throws IOException { + server.close(); + } + + @Test + void basicEvent() { + enqueue(new MockResponse().setBody("data: hey\n\n").setHeader("content-type", "text/event-stream")); + + EventSource source = newEventSource(); + + assertEquals("/", source.request().url().encodedPath()); + + listener.assertOpen(); + listener.assertEvent(null, null, "hey"); + listener.assertClose(); + } + + @Test + void basicRestDataSourceTests() { + assertThrows(UnsupportedOperationException.class, () -> new RestDataSource(null, null)); + assertThrows(UnsupportedOperationException.class, () -> new RestDataSource(null, "")); + assertThrows(IllegalArgumentException.class, () -> new RestDataSource(null, server.url("/sse").toString(), null, "clientName")); // NOSONAR + RestDataSource dataSource = new RestDataSource(null, server.url("/sse").toString()); + assertNotNull(dataSource); + assertDoesNotThrow(dataSource::housekeeping); + } + + @Test + void testRestDataSource() { + try (final ZContext ctx = new ZContext()) { + final RestDataSource dataSource = new RestDataSource(ctx, server.url("/sse").toString()); + assertNotNull(dataSource); + + dataSource.subscribe("1", server.url("/sse").toString(), new byte[0]); + receiveAndCheckData(dataSource, "io.opencmw.client.rest.RestDataSource#*", true); + + // test asynchronuous get + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(DEFAULT_WAIT_MILLIS)); + dataSource.enqueueRequest("testHashKey#1"); + final ZMsg returnMessage = receiveAndCheckData(dataSource, "testHashKey#1", true); + assertEquals(0, returnMessage.getLast().getData().length); + + dataSource.stop(); + } + } + + @Test + void testRestDataSourceTimeOut() { + try (final ZContext ctx = new ZContext()) { + final RestDataSource dataSource = new RestDataSource(ctx, server.url("/testDelayed").toString(), Duration.ofMillis(10), "testClient"); + assertNotNull(dataSource); + + // test asynchronuous with time-out + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(DEFAULT_WAIT_MILLIS)); + dataSource.enqueueRequest("testHashKey#1"); + final ZMsg returnMessage = receiveAndCheckData(dataSource, "testHashKey#1", true); + assertNotEquals(0, returnMessage.getLast().getData().length); + + dataSource.stop(); + } + } + + @Test + void testRestDataSourceConnectionError() { + try (final ZContext ctx = new ZContext()) { + final RestDataSource dataSource = new RestDataSource(ctx, server.url("/testError").toString()); + assertNotNull(dataSource); + + // three retries and a successful response + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(DEFAULT_WAIT_MILLIS)); + dataSource.cancelLastCall = 3; // required for unit-testing + dataSource.enqueueRequest("testHashKey#1"); + ZMsg returnMessage = receiveAndCheckData(dataSource, "testHashKey#1", true); + System.err.println("returnMessage = " + returnMessage); + assertNotEquals(0, requireNonNull(requireNonNull(returnMessage.pollLast())).getData().length); + assertNotEquals(0, requireNonNull(requireNonNull(returnMessage.pollLast())).getData().length); + + // four retries without successful response + dataSource.cancelLastCall = 4; // required for unit-testing + dataSource.enqueueRequest("testHashKey#1"); + returnMessage = receiveAndCheckData(dataSource, "testHashKey#1", true); + assertNotEquals(0, requireNonNull(requireNonNull(returnMessage.pollLast())).getData().length); + assertEquals(0, requireNonNull(requireNonNull(returnMessage.pollLast())).getData().length); + + dataSource.stop(); + } + } + + @Test + @Disabled("not to be used in CI/CD environment") + void testLsaRestDataSource() { + try (final ZContext ctx = new ZContext()) { + final String endPoint = "?msg=HalloRaphael;mytime=" + System.currentTimeMillis(); + final RestDataSource dataSource = new RestDataSource(ctx, endPoint); + assertNotNull(dataSource); + dataSource.enqueueRequest("lsaHashKey#1"); + receiveAndCheckData(dataSource, "lsaHashKey#1", false); + + dataSource.stop(); + } + } + + private void enqueue(MockResponse response) { + final Dispatcher dispatcher = server.getDispatcher(); + if (!(dispatcher instanceof CustomDispatcher)) { + throw new IllegalStateException("wrong dispatcher type: " + dispatcher); + } + CustomDispatcher customDispatcher = (CustomDispatcher) dispatcher; + customDispatcher.enquedEvents.offer(response); + } + + private EventSource newEventSource() { + Request.Builder builder = new Request.Builder().url(server.url("/")); + + builder.header("Accept", "event-stream"); + + Request request = builder.build(); + EventSource.Factory factory = EventSources.createFactory(client); + return factory.newEventSource(request, listener); + } + + private ZMsg receiveAndCheckData(final RestDataSource dataSource, final String hashKey, final boolean verbose) { + final ZMQ.Poller poller = dataSource.getCtx().createPoller(1); + final ZMQ.Socket socket = dataSource.getSocket(); + final int socketID = poller.register(socket, ZMQ.Poller.POLLIN); + + final int n = poller.poll(TimeUnit.MILLISECONDS.toMillis(DEFAULT_TIMEOUT_MILLIS)); + assertEquals(1, n, "external socket did not receive the expected number of message frames for hashKey = " + hashKey); + ZMsg msg; + if (poller.pollin(socketID)) { + try { + msg = dataSource.getMessage(); + if (verbose) { + LOGGER.atDebug().addArgument(msg).log("received reply via external socket: '{}'"); + } + + if (msg == null) { + throw new IllegalStateException("no data received"); + } + final String text = msg.getFirst().toString(); + assertTrue(text.matches(hashKey.replace("?", ".?").replace("*", ".*?")), "mesage " + text + " did not match hashKey template " + hashKey); + } catch (ZMQException e) { + final int errorCode = socket.errno(); + LOGGER.atError().setCause(e).addArgument(errorCode).addArgument(ZError.toString(errorCode)).log("recvMsg error {} - {}"); + throw e; + } + } else { + throw new IllegalStateException("no data received - pollin"); + } + + poller.close(); + return msg; + } + + private static class CustomDispatcher extends Dispatcher { + public BlockingQueue enquedEvents = new LinkedBlockingQueue<>(); + @Override + public @NotNull MockResponse dispatch(@NotNull RecordedRequest request) { + if (!enquedEvents.isEmpty()) { + // dispatch enqued events + return enquedEvents.poll(); + } + final String acceptHeader = request.getHeader("Accept"); + final String contentType = request.getHeader("content-type"); + LOGGER.atTrace().addArgument(request).addArgument(request.getPath()).addArgument(contentType).addArgument(acceptHeader) // + .log("server-request: {} path = {} contentType={} accept={}"); + + final String path; + try { + path = request.getPath(); + } catch (NullPointerException e) { + LOGGER.atError().setCause(e).log("server-request exception"); + return new MockResponse().setResponseCode(404); + } + switch (requireNonNull(path)) { + case "/sse": + if ("text/event-stream".equals(acceptHeader)) { + return new MockResponse().setBody("data: event-stream init\n\n").setHeader("content-type", "text/event-stream"); + } + return new MockResponse().setBody(TEST_DATA).setHeader("content-type", MimeType.TEXT.toString()); + case "/test": + return new MockResponse().setResponseCode(200).setBody("special test data"); + case "/testDelayed": + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1000)); + return new MockResponse().setResponseCode(200).setBody("special delayed test data"); + case "/testError": + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(2 * DEFAULT_WAIT_MILLIS)); + return new MockResponse().setResponseCode(200).setBody("special error test data"); + default: + } + return new MockResponse().setResponseCode(404); + } + } +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..7f243cf9 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "*/src/test/**/*" diff --git a/concepts/pom.xml b/concepts/pom.xml new file mode 100644 index 00000000..159f4006 --- /dev/null +++ b/concepts/pom.xml @@ -0,0 +1,35 @@ + + + + opencmw + io.opencmw + ${revision}${sha1}${changelist} + ../pom.xml + + 4.0.0 + + concepts + + Module for experimental features which will be moved to more appropriate modules once they become stable. + + + + + io.opencmw + core + ${revision}${sha1}${changelist} + + + de.gsi.dataset + chartfx-dataset + ${version.chartfx} + + + it.unimi.dsi + fastutil + ${version.fastutil} + + + \ No newline at end of file diff --git a/concepts/src/main/java/io/opencmw/concepts/aggregate/DemuxEventDispatcher.java b/concepts/src/main/java/io/opencmw/concepts/aggregate/DemuxEventDispatcher.java new file mode 100644 index 00000000..da7126ee --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/aggregate/DemuxEventDispatcher.java @@ -0,0 +1,130 @@ +package io.opencmw.concepts.aggregate; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.locks.LockSupport; + +import io.opencmw.utils.Cache; + +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.Sequence; +import com.lmax.disruptor.SequenceReportingEventHandler; +import com.lmax.disruptor.TimeoutHandler; + +/** + * Dispatches aggregation workers upon seeing new values for a specified event field. + * Each aggregation worker then assembles all events for this value and optionally publishes back an aggregated events. + * If the aggregation is not completed within a configurable timeout, a partial AggregationEvent is published. + * + * For now events are aggregated into a list of Objects until a certain number of events is reached. + * The final api should allow to specify different Objects to be placed into a result domain object. + * + * @author Alexander Krimm + */ +public class DemuxEventDispatcher implements SequenceReportingEventHandler { + private static final int N_WORKERS = 4; // number of workers defines the maximum number of aggregate events groups which can be overlapping + private static final long TIMEOUT = 400; + private static final int RETENTION_SIZE = 10; + private static final int N_AGG_ELEMENTS = 3; + private final AggregationHandler[] aggregationHandler; + private final List freeWorkers = Collections.synchronizedList(new ArrayList<>(N_WORKERS)); + private final RingBuffer rb; + // private Map aggregatedBpcts = new SoftHashMap<>(RETENTION_SIZE); + private final Cache aggregatedBpcts = new Cache<>(RETENTION_SIZE); + private Sequence seq; + + public DemuxEventDispatcher(final RingBuffer ringBuffer) { + rb = ringBuffer; + aggregationHandler = new AggregationHandler[N_WORKERS]; + for (int i = 0; i < N_WORKERS; i++) { + aggregationHandler[i] = new AggregationHandler(); + freeWorkers.add(aggregationHandler[i]); + } + } + + public AggregationHandler[] getAggregationHander() { + return aggregationHandler; + } + + @Override + public void onEvent(final TestEventSource.IngestedEvent event, final long nextSequence, final boolean b) { + if (!(event.payload instanceof TestEventSource.Event)) { + return; + } + final long eventBpcts = ((TestEventSource.Event) event.payload).bpcts; + // final boolean alreadyScheduled = Arrays.stream(workers).filter(w -> w.bpcts == eventBpcts).findFirst().isPresent(); + final boolean alreadyScheduled = aggregatedBpcts.containsKey(eventBpcts); + if (alreadyScheduled) { + return; + } + while (true) { + if (!freeWorkers.isEmpty()) { + final AggregationHandler freeWorker = freeWorkers.remove(0); + freeWorker.bpcts = eventBpcts; + freeWorker.aggStart = event.ingestionTime; + aggregatedBpcts.put(eventBpcts, new Object()); // NOPMD - necessary to allocate inside loop + seq.set(nextSequence); // advance sequence to let workers process events up to here + return; + } + // no free worker available + long waitTime = Long.MAX_VALUE; + for (AggregationHandler w : aggregationHandler) { + final long currentTime = System.currentTimeMillis(); + final long diff = currentTime - w.aggStart; + waitTime = Math.min(waitTime, diff * 1_000_000); + if (w.bpcts != -1 && diff < TIMEOUT) { + w.publishAndFreeWorker(true); // timeout reached, publish partial result and free worker + break; + } + } + LockSupport.parkNanos(waitTime); + } + } + + @Override + public void setSequenceCallback(final Sequence sequence) { + this.seq = sequence; + } + + @SuppressWarnings("PMD.AvoidUsingVolatile") // necessary for desired CPU caching behaviour + public class AggregationHandler implements EventHandler, TimeoutHandler { + protected volatile long bpcts = -1; // [ms] + protected volatile long aggStart = -1; // [ns] + private List payloads = new ArrayList<>(); + + @Override + public void onEvent(final TestEventSource.IngestedEvent event, final long sequence, final boolean endOfBatch) { + if (bpcts != -1 && event.ingestionTime > aggStart + TIMEOUT) { + publishAndFreeWorker(true); + return; + } + if (bpcts == -1 || !(event.payload instanceof TestEventSource.Event) || ((TestEventSource.Event) event.payload).bpcts != bpcts) { + return; // skip irrelevant events + } + this.payloads.add(event); + if (payloads.size() == N_AGG_ELEMENTS) { + publishAndFreeWorker(false); + } + } + + protected void publishAndFreeWorker(final boolean partial) { + rb.publishEvent(((event1, sequence1, arg0) -> { + event1.ingestionTime = System.currentTimeMillis(); + event1.payload = partial ? ("aggregation timed out for bpcts: " + bpcts + " -> ") + payloads : payloads; + }), + payloads); + bpcts = -1; + payloads = new ArrayList<>(); + freeWorkers.add(this); + } + + @Override + public void onTimeout(final long sequence) { + if (bpcts != -1 && System.currentTimeMillis() > aggStart + TIMEOUT) { + publishAndFreeWorker(true); + } + } + } +} diff --git a/concepts/src/main/java/io/opencmw/concepts/aggregate/TestEventSource.java b/concepts/src/main/java/io/opencmw/concepts/aggregate/TestEventSource.java new file mode 100644 index 00000000..3aed18b5 --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/aggregate/TestEventSource.java @@ -0,0 +1,155 @@ +package io.opencmw.concepts.aggregate; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.concurrent.locks.LockSupport; + +import com.lmax.disruptor.RingBuffer; + +/** + * An event Source to generate Events with different timing characteristics/orderings. + * + * @author Alexander Krimm + */ +public class TestEventSource implements Runnable { + private static final int DEFAULT_CHAIN = 3; + private static final long DEFAULT_DELTA = 20; + private static final long DEFAULT_PAUSE = 400; + // state for the event source + public final int repeat; + public final String[] eventList; + private final RingBuffer ringBuffer; + + /** + * Generate an event source which plays back the given sequence of events + * + * @param events A string containing a space separated list of events. first letter is type/bpcts, second is number + * Optionally you can add semicolon delimited key=value pairs to assign values in of the events + * @param repeat How often to repeat the given sequence (use zero value for infinite repetition) + * @param rb The ring buffer to publish the event into + */ + public TestEventSource(final String events, final int repeat, final RingBuffer rb) { + eventList = events.split(" "); + this.repeat = repeat; + this.ringBuffer = rb; + } + + @Override + public void run() { + long lastEvent = System.currentTimeMillis(); + long timeOffset = 0; + int repetitionCount = 0; + while (repeat == 0 || repeat > repetitionCount) { + final Iterator eventIterator = Arrays.stream(eventList).iterator(); + while (!Thread.interrupted() && eventIterator.hasNext()) { + final String eventToken = eventIterator.next(); + final String[] tokens = eventToken.split(";"); + if (tokens.length == 0 || tokens[0].isEmpty()) { + continue; + } + if ("pause".equals(tokens[0])) { + lastEvent += DEFAULT_PAUSE; + continue; + } + Event currentEvent = generateEventFromToken(tokens, timeOffset, lastEvent, repetitionCount); + lastEvent = currentEvent.publishTime; + long diff = currentEvent.publishTime - System.currentTimeMillis(); + if (diff > 0) { + LockSupport.parkNanos(1_000_000L * diff); + } + ringBuffer.publishEvent((event, sequence, arg0) -> { + event.ingestionTime = System.currentTimeMillis(); + event.payload = arg0; + }, currentEvent); + } + repetitionCount++; + } + } + + private Event generateEventFromToken(final String[] tokens, final long timeOffset, final long lastEvent, final int repetitionCount) { + String device = tokens[0].substring(0, 1); + long bpcts = Long.parseLong(tokens[0].substring(1)) + repetitionCount * 1000L; + int type = device.charAt(0); + String payload = device + bpcts; + long sourceTime = lastEvent + DEFAULT_DELTA; + long publishTime = sourceTime; + int chain = DEFAULT_CHAIN; + for (int i = 1; i < tokens.length; i++) { + String[] keyvalue = tokens[i].split("="); + if (keyvalue.length != 2) { + continue; + } + switch (keyvalue[0]) { + case "time": + sourceTime = Long.parseLong(keyvalue[1]) + timeOffset; + publishTime = sourceTime; + break; + case "sourceTime": + sourceTime = Long.parseLong(keyvalue[1]) + timeOffset; + break; + case "publishTime": + publishTime = Long.parseLong(keyvalue[1]) + timeOffset; + break; + case "bpcts": + bpcts = Long.parseLong(keyvalue[1]) + repetitionCount * 1000L; + break; + case "chain": + chain = Integer.parseInt(keyvalue[1]); + break; + case "type": + type = Integer.parseInt(keyvalue[1]); + break; + case "device": + device = keyvalue[1]; + break; + case "payload": + payload = keyvalue[1] + "(repetition count: " + repetitionCount + ")"; + break; + default: + throw new IllegalArgumentException("unable to process event keyvalue pair: " + Arrays.toString(keyvalue)); + } + } + return new Event(sourceTime, publishTime, bpcts, chain, type, device, payload); + } + + /** + * Mock event entry. + */ + public static class Event { + public final long sourceTime; + public final long publishTime; + public final long bpcts; + public final int chain; + public final int type; + public final String device; + public final Object payload; + + public Event(final long sourceTime, final long publishTime, final long bpcts, final int chain, final int type, final String device, final Object payload) { + this.sourceTime = sourceTime; + this.publishTime = publishTime; + this.bpcts = bpcts; + this.chain = chain; + this.type = type; + this.device = device; + this.payload = payload; + } + + @Override + public String toString() { + return "Event{sourceTime=" + sourceTime + ", publishTime=" + publishTime + ", bpcts=" + bpcts + ", chain=" + chain + ", type=" + type + ", device='" + device + '\'' + ", payload=" + payload + '}'; + } + } + + /** + * Basic ring buffer event + */ + public static class IngestedEvent { + public long ingestionTime; + public Object payload; + + @Override + public String toString() { + return "IngestedEvent{ingestionTime=" + ingestionTime + ", payload=" + payload + '}'; + } + } +} diff --git a/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoBroker.java b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoBroker.java new file mode 100644 index 00000000..56324f37 --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoBroker.java @@ -0,0 +1,647 @@ +package io.opencmw.concepts.majordomo; + +import static org.zeromq.ZMQ.Socket; + +import static io.opencmw.concepts.majordomo.MajordomoProtocol.*; +import static io.opencmw.concepts.majordomo.MajordomoProtocol.MdpSubProtocol.*; +import static io.opencmw.concepts.majordomo.MajordomoProtocol.MdpWorkerCommand.*; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMQException; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacRole; +import io.opencmw.rbac.RbacToken; +import io.opencmw.utils.SystemProperties; + +/** + * Majordomo Protocol broker -- a minimal implementation of http://rfc.zeromq.org/spec:7 and spec:8 + * + * default heart-beat time-out [ms] is set by system property: 'OpenCMW.heartBeat' // default: 2500 [ms] + * default heart-beat liveness is set by system property: 'OpenCMW.heartBeatLiveness' // [counts] 3-5 is reasonable + * N.B. heartbeat expires when last heartbeat message is more than HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms ago. + * this implies also, that worker must either return their message within 'HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms' or decouple their secondary handler interface into another thread. + * + * default client time-out [s] is set by system property: 'OpenCMW.clientTimeOut' // default: 3600 [s] -- after which unanswered client messages and infos are being deleted + * + */ +@SuppressWarnings({ "PMD.DoNotUseThreads", "PMD.TooManyMethods", "PMD.GodClass", "PMD.UseConcurrentHashMap" }) // this is a concept, HashMap invoked in single-threaded context +public class MajordomoBroker extends Thread { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoBroker.class); + private static final byte[] INTERNAL_SENDER_ID = null; + private static final String INTERNAL_SERVICE_PREFIX = "mmi."; + private static final byte[] INTERNAL_SERVICE_PREFIX_BYTES = INTERNAL_SERVICE_PREFIX.getBytes(StandardCharsets.UTF_8); + private static final long HEARTBEAT_LIVENESS = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeatLiveness", 3); // [counts] 3-5 is reasonable + private static final long HEARTBEAT_INTERVAL = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeat", 2500); // [ms] + private static final long HEARTBEAT_EXPIRY = HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS; + private static final int CLIENT_TIMEOUT = SystemProperties.getValueIgnoreCase("OpenCMW.clientTimeOut", 0); // [s] + private static final AtomicInteger BROKER_COUNTER = new AtomicInteger(); + private static final AtomicInteger WORKER_COUNTER = new AtomicInteger(); + + // --------------------------------------------------------------------- + private final ZContext ctx; + private final Socket internalRouterSocket; + private final Socket internalServiceSocket; + private final List routerSockets = new ArrayList<>(); // Sockets for clients & public external workers + private final AtomicBoolean run = new AtomicBoolean(false); // NOPMD + private final SortedSet> rbacRoles; + private final Map services = new HashMap<>(); // known services Map<'service name', Service> + private final Map workers = new HashMap<>(); // known workers Map + private final Map clients = new HashMap<>(); + + private final Deque waiting = new ArrayDeque<>(); // idle workers + private long heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; // When to send HEARTBEAT + + /** + * Initialize broker state. + * @param ioThreads number of threads dedicated to network IO (recommendation 1 thread per 1 GBit/s) + * @param rbacRoles RBAC-based roles (used for IO prioritisation and service access control + */ + public MajordomoBroker(final int ioThreads, final RbacRole... rbacRoles) { + super(); + this.setName(MajordomoBroker.class.getSimpleName() + "#" + BROKER_COUNTER.getAndIncrement()); + + ctx = new ZContext(ioThreads); + + // initialise RBAC role-based priority queues + this.rbacRoles = Collections.unmodifiableSortedSet(new TreeSet<>(Set.of(rbacRoles))); + + // generate and register internal default inproc socket + this.internalRouterSocket = bind("inproc://broker"); // NOPMD + this.internalServiceSocket = bind("inproc://intService"); // NOPMD + //this.internalServiceSocket.setRouterMandatory(true); + + registerDefaultServices(rbacRoles); // NOPMD + } + + public void addInternalService(final MajordomoWorker worker, final int nServiceThreads) { + assert worker != null : "worker must not be null"; + final Service oldWorker = services.put(worker.getServiceName(), new Service(worker.getServiceName(), worker.getServiceName().getBytes(StandardCharsets.UTF_8), worker, nServiceThreads)); + if (oldWorker != null) { + LOGGER.atWarn().addArgument(worker.getServiceName()).log("overwriting existing internal service definition for '{}'"); + } + } + + /** + * Bind broker to endpoint, can call this multiple times. We use a single + * socket for both clients and workers. + */ + public Socket bind(String endpoint) { + final Socket routerSocket = ctx.createSocket(SocketType.ROUTER); + routerSocket.setHWM(0); + routerSocket.bind(endpoint); + routerSockets.add(routerSocket); + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(endpoint).log("Majordomo broker/0.1 is active at '{}'"); + } + return routerSocket; + } + + public ZContext getContext() { + return ctx; + } + + public Socket getInternalRouterSocket() { + return internalRouterSocket; + } + + /** + * @return unmodifiable list of registered external sockets + */ + public List getRouterSockets() { + return Collections.unmodifiableList(routerSockets); + } + + public Collection getServices() { + return services.values(); + } + + public boolean isRunning() { + return run.get(); + } + + public void removeService(final String serviceName) { + services.remove(serviceName); + } + + /** + * main broker work happens here + */ + @Override + public void run() { + try (ZMQ.Poller items = ctx.createPoller(routerSockets.size())) { + for (Socket routerSocket : routerSockets) { // NOPMD - closed below in finally + items.register(routerSocket, ZMQ.Poller.POLLIN); + } + while (run.get() && !Thread.currentThread().isInterrupted()) { + if (items.poll(HEARTBEAT_INTERVAL) == -1) { + break; // interrupted + } + + int loopCount = 0; + while (run.get()) { + boolean processData = false; + for (Socket routerSocket : routerSockets) { // NOPMD - closed below in finally + final MdpMessage msg = receiveMdpMessage(routerSocket, false); + if (msg != null) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(msg).log("Majordomo broker received new message: '{}'"); + } + processData |= handleReceivedMessage(routerSocket, msg); + } + } + + processClients(); + if (loopCount % 10 == 0) { + // perform maintenance tasks during the first and every tenth iteration + purgeWorkers(); + purgeClients(); + sendHeartbeats(); + } + loopCount++; + if (!processData) { + break; + } + } + } + + } finally { + routerSockets.forEach(Socket::close); + } + destroy(); // interrupted + } + + @Override + public synchronized void start() { // NOPMD - need to be synchronised on class level due to super definition + run.set(true); + services.forEach((serviceName, service) -> service.internalWorkers.forEach(Thread::start)); + super.start(); + } + + public void stopBroker() { + run.set(false); + } + + /** + * Deletes worker from all data structures, and destroys worker. + */ + protected void deleteWorker(Worker worker, boolean disconnect) { + assert (worker != null); + if (disconnect) { + sendWorkerMessage(worker.socket, W_DISCONNECT, worker.address, null); + } + if (worker.service != null) { + worker.service.waiting.remove(worker); + } + workers.remove(worker.addressHex); + } + + /** + * Disconnect all workers, destroy context. + */ + protected void destroy() { + Worker[] deleteList = workers.values().toArray(new Worker[0]); + for (Worker worker : deleteList) { + deleteWorker(worker, true); + } + ctx.destroy(); + } + + /** + * Dispatch requests to waiting workers as possible + */ + protected void dispatch(Service service) { + assert (service != null); + purgeWorkers(); + while (!service.waiting.isEmpty() && service.requestsPending()) { + final MdpMessage msg = service.getNextPrioritisedMessage(); + if (msg == null) { + // should be thrown only with VM '-ea' enabled -- assert noisily since this a (rare|design) library error + assert false : "getNextPrioritisedMessage should not be null"; + continue; + } + Worker worker = service.waiting.pop(); + waiting.remove(worker); + sendWorkerMessage(worker.socket, W_REQUEST, worker.address, msg.senderID, msg.payload); + } + } + + protected boolean handleReceivedMessage(final Socket receiveSocket, final MdpMessage msg) { + switch (msg.protocol) { + case C_CLIENT: + // Set reply return address to client sender + final Client client = clients.computeIfAbsent(msg.senderName, s -> new Client(receiveSocket, msg.protocol, msg.senderName, msg.senderID)); + client.offerToQueue((MdpClientMessage) msg); + return true; + case W_WORKER: + processWorker(receiveSocket, (MdpWorkerMessage) msg); + return true; + default: + // N.B. not too verbose logging since we do not want that sloppy clients can bring down the broker through warning or info messages + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(msg).log("Majordomo broker invalid message: '{}'"); + } + return false; + } + } + + /** + * Process a request coming from a client. + */ + protected void processClients() { + // round-robbin + clients.forEach((name, client) -> { + final MdpClientMessage clientMessage = client.pop(); + if (clientMessage == null) { + return; + } + // dispatch client message to worker queue + final Service service = services.get(clientMessage.serviceName); + if (service == null) { + // not implemented -- according to Majordomo Management Interface (MMI) as defined in http://rfc.zeromq.org/spec:8 + sendClientMessage(client.socket, MdpClientCommand.C_UNKNOWN, clientMessage.senderID, clientMessage.serviceNameBytes, "501".getBytes(StandardCharsets.UTF_8)); + return; + } + // queue new client message RBAC-priority-based + service.putPrioritisedMessage(clientMessage); + + // dispatch service + if (service.isInternal) { + final MdpClientMessage msg = service.getNextPrioritisedMessage(); + if (msg == null) { + // should be thrown only with VM '-ea' enabled -- assert noisily since this a (rare|design) library error + assert false : "getNextPrioritisedMessage should not be null"; + return; + } + sendWorkerMessage(service.internalDispatchSocket, W_REQUEST, null, msg.senderID, msg.payload); + } else { + //dispatch(requireService(clientMessage.serviceName, clientMessage.serviceNameBytes)); + dispatch(service); + } + }); + } + + /** + * Process message sent to us by a worker. + */ + protected void processWorker(final Socket receiveSocket, final MdpWorkerMessage msg) { + final boolean isInternal = internalServiceSocket.equals(receiveSocket); + final boolean workerReady = isInternal || workers.containsKey(msg.senderIdHex); + final Worker worker; // = requireWorker(receiveSocket, msg.senderID, msg.senderIdHex); + switch (msg.command) { + case W_READY: + worker = requireWorker(receiveSocket, msg.senderID, msg.senderIdHex); + // Not first mdpWorkerCommand in session || Reserved service name + if (workerReady || Arrays.equals(INTERNAL_SERVICE_PREFIX_BYTES, 0, 3, msg.senderID, 0, 3)) { + deleteWorker(worker, true); + } else { + // Attach worker to service and mark as idle + worker.service = requireService(msg.serviceName, msg.serviceNameBytes); + workerWaiting(worker); + } + break; + case W_REPLY: + if (workerReady) { + worker = isInternal ? null : requireWorker(receiveSocket, msg.senderID, msg.senderIdHex); + final byte[] serviceName = isInternal ? msg.senderID : worker.service.nameBytes; + final Client client = clients.get(msg.clientSourceName); + if (client == null || client.socket == null) { + break; + } + + if (client.protocol == C_CLIENT) { // OpenCMW + sendClientMessage(client.socket, MdpClientCommand.C_UNKNOWN, msg.clientSourceID, serviceName, msg.payload); + } else { + // TODO: add other branches for: + // * CmwLight + // * REST/JSON + // * REST/HTML + throw new IllegalStateException("Unexpected value: " + client.protocol); + } + if (!isInternal) { + workerWaiting(worker); + } + } else { + worker = requireWorker(receiveSocket, msg.senderID, msg.senderIdHex); + deleteWorker(worker, true); + } + break; + case W_HEARTBEAT: + worker = requireWorker(receiveSocket, msg.senderID, msg.senderIdHex); + if (workerReady) { + worker.expiry = System.currentTimeMillis() + HEARTBEAT_EXPIRY; + } else { + deleteWorker(worker, true); + } + break; + case W_DISCONNECT: + worker = requireWorker(receiveSocket, msg.senderID, msg.senderIdHex); + deleteWorker(worker, false); + break; + default: + // N.B. not too verbose logging since we do not want that sloppy clients can bring down the broker through warning or info messages + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(msg).log("Majordomo broker invalid message: '{}'"); + } + break; + } + } + + /** + * Look for & kill expired clients. + */ + protected void purgeClients() { + if (CLIENT_TIMEOUT <= 0) { + return; + } + for (String clientName : clients.keySet()) { // copy because we are going to remove keys + Client client = clients.get(clientName); + if (client == null || client.expiry < System.currentTimeMillis()) { + clients.remove(clientName); + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(client).log("Majordomo broker deleting expired client: '{}'"); + } + } + } + } + + /** + * Look for & kill expired workers. Workers are oldest to most recent, so we stop at the first alive worker. + */ + protected /*synchronized*/ void purgeWorkers() { + for (Worker w = waiting.peekFirst(); w != null && w.expiry < System.currentTimeMillis(); w = waiting.peekFirst()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.atInfo().addArgument(w.addressHex).addArgument(w.service == null ? "(unknown)" : w.service.name).log("Majordomo broker deleting expired worker: '{}' - service: '{}'"); + } + deleteWorker(waiting.pollFirst(), false); + } + } + + protected void registerDefaultServices(final RbacRole[] rbacRoles) { + // add simple internal Majordomo worker + + // Majordomo Management Interface (MMI) as defined in http://rfc.zeromq.org/spec:8 + MajordomoWorker mmiService = new MajordomoWorker(ctx, "mmi.service", rbacRoles); + mmiService.registerHandler(payload -> { + final String serviceName = new String(payload[0], StandardCharsets.UTF_8); + final String returnCode = services.containsKey(serviceName) ? "200" : "400"; + return new byte[][] { returnCode.getBytes(StandardCharsets.UTF_8) }; + }); + addInternalService(mmiService, 1); + + // echo service + MajordomoWorker echoService = new MajordomoWorker(ctx, "mmi.echo", rbacRoles); + echoService.registerHandler(input -> input); // output = input : echo service is complex :-) + addInternalService(echoService, 1); + } + + /** + * Locates the service (creates if necessary). + * + * @param serviceName service name + * @param serviceNameBytes UTF-8 encoded service name + */ + protected Service requireService(final String serviceName, final byte[] serviceNameBytes) { + assert (serviceNameBytes != null); + return services.computeIfAbsent(serviceName, s -> new Service(serviceName, serviceNameBytes, null, 0)); + } + + /** + * Finds the worker (creates if necessary). + */ + protected Worker requireWorker(final Socket socket, final byte[] address, final String addressHex) { + assert (addressHex != null); + return workers.computeIfAbsent(addressHex, identity -> { + if (LOGGER.isInfoEnabled()) { + LOGGER.atInfo().addArgument(addressHex).log("Majordomo broker registering new worker: '{}'"); + } + return new Worker(socket, address, addressHex); + }); + } + + /** + * Send heartbeats to idle workers if it's time + */ + protected /*synchronized*/ void sendHeartbeats() { + // Send heartbeats to idle workers if it's time + if (System.currentTimeMillis() >= heartbeatAt) { + for (Worker worker : waiting) { + sendWorkerMessage(worker.socket, W_HEARTBEAT, worker.address, null); + } + heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; + } + } + + /** + * This worker is now waiting for work. + */ + protected /*synchronized*/ void workerWaiting(Worker worker) { + // Queue to broker and service waiting lists + waiting.addLast(worker); + //TODO: evaluate addLast vs. push (addFirst) - latter should be more beneficial w.r.t. CPU context switches (reuses the same thready/context frequently + // do not know why original implementation wanted to spread across different workers (load balancing across different machines perhaps?!=) + //worker.service.waiting.addLast(worker); + worker.service.waiting.push(worker); + worker.expiry = System.currentTimeMillis() + HEARTBEAT_EXPIRY; + dispatch(worker.service); + } + + /** + * Main method - create and start new broker. + * + * @param args use '-v' for putting worker in verbose mode + */ + public static void main(String[] args) { + MajordomoBroker broker = new MajordomoBroker(1, BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + // Can be called multiple times with different endpoints + broker.bind("tcp://*:5555"); + broker.bind("tcp://*:5556"); + + for (int i = 0; i < 10; i++) { + // simple internalSock echo + MajordomoWorker workerSession = new MajordomoWorker(broker.getContext(), "inproc.echo", BasicRbacRole.ADMIN); // NOPMD - necessary to allocate inside loop + workerSession.registerHandler(input -> input); // output = input : echo service is complex :-) + workerSession.start(); + } + + broker.start(); + } + + /** + * This defines a client service. + */ + protected static class Client { + protected final Socket socket; // Socket worker is connected to + protected final MdpSubProtocol protocol; + protected final String name; // Service name + protected final byte[] nameBytes; // Service name as byte array + protected final String nameHex; // Service name as hex String + private final Deque requests = new ArrayDeque<>(); // List of client requests + protected long expiry = System.currentTimeMillis() + CLIENT_TIMEOUT * 1000L; // Expires at unless heartbeat + + public Client(final Socket socket, final MdpSubProtocol protocol, final String name, final byte[] nameBytes) { + this.socket = socket; + this.protocol = protocol; + this.name = name; + this.nameBytes = nameBytes == null ? name.getBytes(StandardCharsets.UTF_8) : nameBytes; + this.nameHex = strhex(nameBytes); + } + + public void offerToQueue(final MdpClientMessage msg) { + expiry = System.currentTimeMillis() + CLIENT_TIMEOUT * 1000L; + requests.offer(msg); + } + + public MdpClientMessage pop() { + return requests.isEmpty() ? null : requests.poll(); + } + } + + /** + * This defines a single service. + */ + protected class Service { + protected final String name; // Service name + protected final byte[] nameBytes; // Service name as byte array + protected final MajordomoWorker mdpWorker; + protected final boolean isInternal; + protected final Map, Queue> requests = new HashMap<>(); // NOPMD RBAC-based queuing - thread-safe use of HashMap + protected final Deque waiting = new ArrayDeque<>(); // List of waiting workers + protected final List internalWorkers = new ArrayList<>(); + protected final Socket internalDispatchSocket; + + public Service(final String name, final byte[] nameBytes, final MajordomoWorker mdpWorker, final int nInternalThreads) { + this.name = name; + this.nameBytes = nameBytes == null ? name.getBytes(StandardCharsets.UTF_8) : nameBytes; + this.mdpWorker = mdpWorker; + this.isInternal = mdpWorker != null; + if (isInternal) { + this.internalDispatchSocket = ctx.createSocket(SocketType.PUSH); + this.internalDispatchSocket.setHWM(0); + this.internalDispatchSocket.bind("inproc://" + mdpWorker.getServiceName() + "push"); + for (int i = 0; i < nInternalThreads; i++) { + internalWorkers.add(new InternalWorkerThread(this)); // NOPMD - necessary to allocate inside loop + } + } else { + this.internalDispatchSocket = null; + } + rbacRoles.forEach(role -> requests.put(role, new ArrayDeque<>())); + requests.put(BasicRbacRole.NULL, new ArrayDeque<>()); // add default queue + } + + public boolean requestsPending() { + return requests.entrySet().stream().anyMatch(map -> !map.getValue().isEmpty()); + } + + /** + * @return next RBAC prioritised message or 'null' if there aren't any + */ + protected final MdpClientMessage getNextPrioritisedMessage() { + for (RbacRole role : rbacRoles) { + final Queue queue = requests.get(role); // matched non-empty queue + if (!queue.isEmpty()) { + return queue.poll(); + } + } + final Queue queue = requests.get(BasicRbacRole.NULL); // default queue + return queue.isEmpty() ? null : queue.poll(); + } + + protected void putPrioritisedMessage(final MdpClientMessage queuedMessage) { + if (queuedMessage.hasRbackToken()) { + // find proper RBAC queue + final RbacToken rbacToken = RbacToken.from(queuedMessage.getRbacFrame()); + final Queue roleBasedQueue = requests.get(rbacToken.getRole()); + if (roleBasedQueue != null) { + roleBasedQueue.offer(queuedMessage); + } + } else { + requests.get(BasicRbacRole.NULL).offer(queuedMessage); + } + } + } + + /** + * This defines one worker, idle or active. + */ + protected class Worker { + protected final Socket socket; // Socket worker is connected to + protected final byte[] address; // Address ID frame to route to + protected final String addressHex; // Address ID frame of worker expressed as hex-String + + protected final boolean external; + protected Service service; // Owning service, if known + protected long expiry = System.currentTimeMillis() + HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS; // Expires at unless heartbeat + + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + public Worker(final Socket socket, final byte[] address, final String addressHex) { + this.socket = socket; + this.external = true; + this.address = address; + this.addressHex = addressHex; + } + } + + protected class InternalWorkerThread extends Thread { + private final Service service; + + public InternalWorkerThread(final Service service) { + super(); + assert service != null && service.name != null && !service.name.isBlank(); + final String serviceName = service.name + "#" + WORKER_COUNTER.getAndIncrement(); + this.setName(MajordomoBroker.class.getSimpleName() + "-" + InternalWorkerThread.class.getSimpleName() + ":" + serviceName); + this.setDaemon(true); + this.service = service; + } + + @Override + public void run() { + try (Socket sendSocket = ctx.createSocket(SocketType.DEALER); + Socket receiveSocket = ctx.createSocket(SocketType.PULL); + ZMQ.Poller items = ctx.createPoller(1)) { + // register worker with broker + sendSocket.setSndHWM(0); + sendSocket.setIdentity(service.name.getBytes(StandardCharsets.UTF_8)); + sendSocket.connect("inproc://intService"); + + receiveSocket.connect("inproc://" + service.mdpWorker.getServiceName() + "push"); + receiveSocket.setRcvHWM(0); + + // register poller + items.register(receiveSocket, ZMQ.Poller.POLLIN); + while (run.get() && !this.isInterrupted()) { + if (items.poll(HEARTBEAT_INTERVAL) == -1) { + break; // interrupted + } + + while (run.get()) { + final MdpMessage mdpMessage = receiveMdpMessage(receiveSocket, false); + if (mdpMessage == null || mdpMessage.isClient) { + break; + } + MdpWorkerMessage msg = (MdpWorkerMessage) mdpMessage; + + // execute the user-provided call-back function + final MdpMessage reply = service.mdpWorker.processRequest(msg, msg.clientSourceID); + + if (reply != null) { + sendWorkerMessage(sendSocket, W_REPLY, INTERNAL_SENDER_ID, msg.clientSourceID, reply.payload); + } + } + } + } catch (ZMQException e) { + // process should abort + } + } + } +} diff --git a/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoClientV1.java b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoClientV1.java new file mode 100644 index 00000000..30ecfe1a --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoClientV1.java @@ -0,0 +1,163 @@ +package io.opencmw.concepts.majordomo; + +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Formatter; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; +import org.zeromq.util.ZData; + +/** +* Majordomo Protocol Client API, Java version Implements the OpenCmwProtocol/Worker spec at +* http://rfc.zeromq.org/spec:7. +* +*/ +public class MajordomoClientV1 { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoClientV1.class); + private static final AtomicInteger CLIENT_V1_INSTANCE = new AtomicInteger(); + private final String uniqueID; + private final byte[] uniqueIdBytes; + private final String broker; + private final ZContext ctx; + private ZMQ.Socket clientSocket; + private long timeout = 2500; + private int retries = 3; + private final Formatter log = new Formatter(System.out); + private ZMQ.Poller poller; + + public MajordomoClientV1(String broker, String clientName) { + this.broker = broker; + ctx = new ZContext(); + + uniqueID = clientName + "PID=" + ManagementFactory.getRuntimeMXBean().getName() + "-InstanceID=" + CLIENT_V1_INSTANCE.getAndIncrement(); + uniqueIdBytes = uniqueID.getBytes(ZMQ.CHARSET); + + reconnectToBroker(); + } + + /** + * Connect or reconnect to broker + */ + private void reconnectToBroker() { + if (clientSocket != null) { + clientSocket.close(); + } + clientSocket = ctx.createSocket(SocketType.DEALER); + clientSocket.setHWM(0); + clientSocket.setIdentity(uniqueIdBytes); + clientSocket.connect(broker); + + if (poller != null) { + poller.unregister(clientSocket); + poller.close(); + } + poller = ctx.createPoller(1); + poller.register(clientSocket, ZMQ.Poller.POLLIN); + LOGGER.atDebug().addArgument(broker).log("connecting to broker at: '{}'"); + } + + public void destroy() { + ctx.destroy(); + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public String getUniqueID() { + return uniqueID; + } + + /** + * Send request to broker and get reply by hook or crook takes ownership of + * request message and destroys it when sent. Returns the reply message or + * NULL if there was no reply. + * + * @param service service name + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + * @return reply message or NULL if there was no reply + */ + public ZMsg send(final String service, final byte[]... msgs) { + return send(service.getBytes(StandardCharsets.UTF_8), msgs); + } + + /** + * Send request to broker and get reply by hook or crook takes ownership of + * request message and destroys it when sent. Returns the reply message or + * NULL if there was no reply. + * + * @param service UTF-8 encoded service name byte array + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + * @return reply message or NULL if there was no reply + */ + public ZMsg send(final byte[] service, final byte[]... msgs) { + ZMsg reply = null; + + int retriesLeft = retries; + while (retriesLeft > 0 && !Thread.currentThread().isInterrupted()) { + if (!MajordomoProtocol.sendClientMessage(clientSocket, MajordomoProtocol.MdpClientCommand.C_UNKNOWN, null, service, msgs)) { + throw new IllegalStateException("could not send request " + Arrays.stream(msgs).map(ZData::toString).collect(Collectors.joining(",", "[", "]"))); + } + + // Poll socket for a reply, with timeout + if (poller.poll(timeout) == -1) { + break; // Interrupted + } + + if (poller.pollin(0)) { + ZMsg msg = ZMsg.recvMsg(clientSocket, false); + LOGGER.atDebug().addArgument(msg).log("received reply: '{}'"); + + if (msg == null) { + break; + } + // Don't try to handle errors, just assert noisily + assert (msg.size() >= 3); + + ZFrame emptyFrame = msg.pop(); + assert emptyFrame.size() == 0; + + ZFrame header = msg.pop(); + assert (MajordomoProtocol.MdpSubProtocol.C_CLIENT.isEquals(header.getData())); + header.destroy(); + + ZFrame replyService = msg.pop(); + //noinspection AssertWithSideEffects + assert Arrays.equals(service, replyService.getData()); // NOSONAR NOPMD only read, not mutation of service + replyService.destroy(); + + reply = msg; + break; + } else { + if (--retriesLeft == 0) { + log.format("W: permanent error, abandoning\n"); + break; + } + log.format("W: no reply, reconnecting\n"); + reconnectToBroker(); + } + } + return reply; + } +} diff --git a/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoClientV2.java b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoClientV2.java new file mode 100644 index 00000000..a5c18bf1 --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoClientV2.java @@ -0,0 +1,118 @@ +package io.opencmw.concepts.majordomo; + +import static io.opencmw.concepts.majordomo.MajordomoProtocol.*; + +import java.nio.charset.StandardCharsets; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +/** + * Majordomo Protocol Client API, asynchronous Java version. Implements the + * OpenCmwProtocol/Worker spec at http://rfc.zeromq.org/spec:7. + */ +public class MajordomoClientV2 { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoClientV2.class); + private final String broker; + private final ZContext ctx; + private ZMQ.Socket clientSocket; + private long timeout = 2500; + private ZMQ.Poller poller; + + public MajordomoClientV2(final String broker) { + this.broker = broker; + ctx = new ZContext(); + reconnectToBroker(); + } + + public void destroy() { + ctx.destroy(); + } + + public long getTimeout() { + return timeout; + } + + /** + * Returns the reply message or NULL if there was no reply. Does not attempt + * to recover from a broker failure, this is not possible without storing + * all unanswered requests and resending them all… + */ + public ZMsg recv() { + // Poll socket for a reply, with timeout + if (poller.poll(timeout * 1000) == -1) { + return null; // Interrupted + } + + if (poller.pollin(0)) { + ZMsg msg = ZMsg.recvMsg(clientSocket); + LOGGER.atDebug().addArgument(msg).log("received reply: '{}'"); + + // Don't try to handle errors, just assert noisily + assert (msg.size() >= 4); + + ZFrame empty = msg.pop(); + assert (empty.getData().length == 0); + empty.destroy(); + + ZFrame header = msg.pop(); + assert (MdpSubProtocol.C_CLIENT.isEquals(header.getData())); + header.destroy(); + + ZFrame replyService = msg.pop(); + replyService.destroy(); + + return msg; + } + return null; + } + + /** + * Send request to broker and get reply by hook or crook Takes ownership of request message and destroys it when sent. + * + * @param service UTF-8 encoded service name byte array + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + */ + public boolean send(final byte[] service, final byte[]... msgs) { + return sendClientMessage(clientSocket, MdpClientCommand.C_UNKNOWN, null, service, msgs); + } + + /** + * Send request to broker and get reply by hook or crook Takes ownership of request message and destroys it when sent. + * + * @param service UTF-8 encoded service name byte array + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + */ + public boolean send(final String service, final byte[]... msgs) { + return send(service.getBytes(StandardCharsets.UTF_8), msgs); + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + /** + * Connect or reconnect to broker + */ + private void reconnectToBroker() { + if (clientSocket != null) { + clientSocket.close(); + } + clientSocket = ctx.createSocket(SocketType.DEALER); + clientSocket.setHWM(0); + clientSocket.setIdentity("clientV2".getBytes(StandardCharsets.UTF_8)); + clientSocket.connect(broker); + if (poller != null) { + poller.unregister(clientSocket); + poller.close(); + } + poller = ctx.createPoller(1); + poller.register(clientSocket, ZMQ.Poller.POLLIN); + LOGGER.atDebug().addArgument(broker).log("connecting to broker at: '{}'"); + } +} diff --git a/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoProtocol.java b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoProtocol.java new file mode 100644 index 00000000..3a4933f8 --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoProtocol.java @@ -0,0 +1,423 @@ +package io.opencmw.concepts.majordomo; + +import static org.zeromq.ZMQ.Socket; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.zeromq.SocketType; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; +import org.zeromq.util.ZData; + +import zmq.SocketBase; + +/** + * Majordomo Protocol (MDP) definitions and implementations according to https://rfc.zeromq.org/spec/7/ + * + * @author rstein + */ +@SuppressWarnings({ "PMD.ArrayIsStoredDirectly", "PMD.MethodReturnsInternalArray" }) +public class MajordomoProtocol { // NOPMD nomen-est-omen + public static final byte[] EMPTY_FRAME = {}; + private static final String HEX_CHAR = "0123456789ABCDEF"; + + public static MdpMessage receiveMdpMessage(final Socket socket) { + return receiveMdpMessage(socket, true); + } + + public static MdpMessage receiveMdpMessage(final Socket socket, final boolean wait) { //NOPMD + assert socket != null : "socket must not be null"; + final int flags = wait ? 0 : ZMQ.DONTWAIT; + + final ZMsg msg = ZMsg.recvMsg(socket, flags); + if (msg == null) { + return null; + } + assert msg.size() >= 3; + + final byte[] senderId; + + if (socket.getSocketType() == SocketType.ROUTER) { + senderId = msg.pop().getData(); + assert senderId != null : "first sender frame is empty"; + } else { + senderId = null; + } + + final ZFrame emptyFrame = msg.pop(); + assert emptyFrame.hasData() && emptyFrame.getData().length == 0 : "nominally empty message has data: " + emptyFrame.getData().length + " - '" + emptyFrame.toString() + "'"; + + final ZFrame protocolFrame = msg.pop(); + assert protocolFrame.hasData(); + final MdpSubProtocol protocol = MdpSubProtocol.getProtocol(protocolFrame); + + switch (protocol) { + case C_CLIENT: + assert msg.size() >= 2; + final MdpClientCommand clientCommand = MdpClientCommand.getCommand(MdpClientCommand.C_UNKNOWN.newFrame()); + final ZFrame serviceName = msg.pop(); + assert serviceName.getData() != null : "empty serviceName"; + + final byte[][] clientMessages = new byte[msg.size()][]; + for (int i = 0; i < clientMessages.length; i++) { + final ZFrame dataFrame = msg.pop(); + clientMessages[i] = dataFrame.hasData() ? dataFrame.getData() : EMPTY_FRAME; + dataFrame.destroy(); + } + return new MdpClientMessage(senderId, clientCommand, serviceName.getData(), clientMessages); + case W_WORKER: + final ZFrame commandFrame = msg.pop(); + assert protocolFrame.hasData(); + final MdpWorkerCommand workerCommand = MdpWorkerCommand.getCommand(commandFrame); + switch (workerCommand) { + case W_HEARTBEAT: + case W_DISCONNECT: + assert msg.isEmpty() + : "MDP V0.1 does not support further frames for W_HEARTBEAT or W_DISCONNECT"; + return new MdpWorkerMessage(senderId, workerCommand, null, null); + case W_READY: + // service-name is optional + return new MdpWorkerMessage(senderId, workerCommand, msg.isEmpty() ? null : msg.pop().getData(), null); + case W_REQUEST: + case W_REPLY: + final byte[] clientSourceId = msg.pop().getData(); + assert clientSourceId != null : "clientSourceID must not be null"; + final ZFrame emptyFrame2 = msg.pop(); + assert emptyFrame2.hasData() && emptyFrame2.getData().length == 0 : "nominally empty message has data: " + emptyFrame2.getData().length + " - '" + emptyFrame2.toString() + "'"; + + final byte[][] workerMessages = new byte[msg.size()][]; + for (int i = 0; i < workerMessages.length; i++) { + final ZFrame dataFrame = msg.pop(); + workerMessages[i] = dataFrame.hasData() ? dataFrame.getData() : EMPTY_FRAME; + dataFrame.destroy(); + } + return new MdpWorkerMessage(senderId, workerCommand, null, clientSourceId, workerMessages); + + case W_UNKNOWN: + default: + assert false : "should not reach here for production code"; + return null; + } + case UNKNOWN: + default: + assert false : "should not reach here for production code"; + return null; + } + } + + /** + * Send worker message according to the MDP 'client' sub-protocol + * + * @param socket ZeroMQ socket to send the message on + * @param mdpClientCommand the OpenCmwProtocol mdpWorkerCommand + * @param sourceID the unique source ID of the peer client (usually 5 bytes, can be overwritten, BROKER sockets need this to be non-null) + * @param serviceName the unique, original source ID the broker shall forward this message to + * @param msg message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + * + * @return {@code true} if successful + */ + public static boolean sendClientMessage(final Socket socket, final MdpClientCommand mdpClientCommand, final byte[] sourceID, final byte[] serviceName, final byte[]... msg) { + assert socket != null : "socket must not be null"; + assert mdpClientCommand != null : "mdpClientCommand must not be null"; + assert serviceName != null : "serviceName must not be null"; + + final SocketBase socketBase = socket.base(); + boolean status = true; + if (socket.getSocketType() == SocketType.ROUTER) { + assert sourceID != null : "sourceID must be non-null when using ROUTER sockets"; + status = socketBase.send(new zmq.Msg(sourceID), ZMQ.SNDMORE); // frame 0: source ID (optional, only needed for broker sockets) + } + status &= socketBase.send(new zmq.Msg(EMPTY_FRAME), ZMQ.SNDMORE); // frame 1: empty frame (0 bytes) + status &= socketBase.send(new zmq.Msg(MdpSubProtocol.C_CLIENT.getFrameData()), ZMQ.SNDMORE); // frame 2: 'MDPCxx' client sub-protocol version + status &= socketBase.send(new zmq.Msg(serviceName), ZMQ.SNDMORE); // frame 3: service name (UTF-8 string) + // frame 3++: msg frames (last one being usually the RBAC token) + for (int i = 0; i < msg.length; i++) { + status &= socketBase.send(new zmq.Msg(msg[i]), i < msg.length - 1 ? ZMQ.SNDMORE : 0); // NOPMD - necessary to allocate inside loop + } + + return status; + } + + /** + * Send worker message according to the MDP 'worker' sub-protocol + * + * @param socket ZeroMQ socket to send the message on + * @param mdpWorkerCommand the OpenCmwProtocol mdpWorkerCommand + * @param sourceID the unique source ID of the peer client (usually 5 bytes, can be overwritten, BROKER sockets need this to be non-null) + * @param clientID the unique, original source ID the broker shall forward this message to + * @param msg message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + * + * @return {@code true} if successful + */ + public static boolean sendWorkerMessage(final Socket socket, MdpWorkerCommand mdpWorkerCommand, final byte[] sourceID, final byte[] clientID, final byte[]... msg) { + assert socket != null : "socket must not be null"; + assert mdpWorkerCommand != null : "mdpWorkerCommand must not be null"; + + final SocketBase socketBase = socket.base(); + boolean status = true; + if (socket.getSocketType() == SocketType.ROUTER) { + assert sourceID != null : "sourceID must be non-null when using ROUTER sockets"; + status = socketBase.send(new zmq.Msg(sourceID), ZMQ.SNDMORE); // frame 0: source ID (optional, only needed for broker sockets) + } + socketBase.send(new zmq.Msg(EMPTY_FRAME), ZMQ.SNDMORE); // frame 1: empty frame (0 bytes) + socketBase.send(new zmq.Msg(MdpSubProtocol.W_WORKER.getFrameData()), ZMQ.SNDMORE); // frame 2: 'MDPWxx' worker sub-protocol version + + switch (mdpWorkerCommand) { + case W_HEARTBEAT: + case W_DISCONNECT: + assert msg.length == 0 : "byte[]... msg must be empty for W_HEARTBEAT and W_DISCONNECT commands"; + status &= socketBase.send(new zmq.Msg(mdpWorkerCommand.getFrameData()), 0); // frame 3: mdpWorkerCommand (1-byte: W_HEARTBEAT, W_DISCONNECT) + return status; + case W_REQUEST: + case W_REPLY: + case W_READY: + socketBase.send(new zmq.Msg(mdpWorkerCommand.getFrameData()), ZMQ.SNDMORE); // frame 3: mdpWorkerCommand (1-byte: W_READY, W_REQUEST, W_REPLY) + assert clientID != null; + if (msg.length == 0 && mdpWorkerCommand == MdpWorkerCommand.W_READY) { + status &= socketBase.send(new zmq.Msg(clientID), 0); // frame 4: client ID (i.e. sourceID of the client that is known to the broker + } else { + assert msg.length != 0 : "byte[]... msg must not be empty"; + status &= socketBase.send(new zmq.Msg(clientID), ZMQ.SNDMORE); // frame 4: client ID (i.e. sourceID of the client that is known to the broker + + // optional additional payload after ready (e.g. service uniqueID, input/output property layout etc.) + status &= socketBase.send(new zmq.Msg(EMPTY_FRAME), ZMQ.SNDMORE); // frame 5: empty frame (0 bytes) + for (int i = 0; i < msg.length; i++) { + status &= socketBase.send(new zmq.Msg(msg[i]), i < msg.length - 1 ? ZMQ.SNDMORE : 0); // NOPMD - necessary to allocate inside loop + } + } + return status; + case W_UNKNOWN: + default: + throw new IllegalArgumentException("should not reach here/unknown command: " + mdpWorkerCommand); + } + } + + public static String strhex(byte[] data) { + if (data == null) { + return ""; + } + StringBuilder b = new StringBuilder(); + for (byte aData : data) { + int b1 = aData >>> 4 & 0xf; + int b2 = aData & 0xf; + b.append(HEX_CHAR.charAt(b1)); + b.append(HEX_CHAR.charAt(b2)); + } + return b.toString(); + } + + /** + * MDP sub-protocol V0.1 + */ + public enum MdpSubProtocol { + C_CLIENT("MDPC01"), // OpenCmwProtocol/Client implementation version + W_WORKER("MDPW01"), // OpenCmwProtocol/Worker implementation version + UNKNOWN("XXXXXX"); + + private final byte[] data; + MdpSubProtocol(final String value) { + this.data = value.getBytes(StandardCharsets.UTF_8); + } + + public boolean frameEquals(ZFrame frame) { + return Arrays.equals(data, frame.getData()); + } + + public byte[] getFrameData() { + return data; + } + + public boolean isEquals(final byte[] other) { + return Arrays.equals(this.data, other); + } + + public ZFrame newFrame() { + return new ZFrame(data); + } + + public static MdpSubProtocol getProtocol(ZFrame frame) { + for (MdpSubProtocol knownProtocol : MdpSubProtocol.values()) { + if (knownProtocol.frameEquals(frame)) { + if (knownProtocol == UNKNOWN) { + continue; + } + return knownProtocol; + } + } + return UNKNOWN; + } + } + + /** + * OpenCmwProtocol/Server commands, as byte values + */ + public enum MdpWorkerCommand { + W_READY(0x01), + W_REQUEST(0x02), + W_REPLY(0x03), + W_HEARTBEAT(0x04), + W_DISCONNECT(0x05), + W_UNKNOWN(-1); + + private final byte[] data; + MdpWorkerCommand(final int value) { //watch for ints>255, will be truncated + this.data = new byte[] { (byte) (value & 0xFF) }; + } + + public boolean frameEquals(ZFrame frame) { + return Arrays.equals(data, frame.getData()); + } + + public byte[] getFrameData() { + return data; + } + + public ZFrame newFrame() { + return new ZFrame(data); + } + + public static MdpWorkerCommand getCommand(ZFrame frame) { + for (MdpWorkerCommand knownMdpCommand : MdpWorkerCommand.values()) { + if (knownMdpCommand.frameEquals(frame)) { + if (knownMdpCommand == W_UNKNOWN) { + continue; + } + return knownMdpCommand; + } + } + return W_UNKNOWN; + } + } + + /** + * OpenCmwProtocol/Client commands, as byte values + */ + public enum MdpClientCommand { + C_UNKNOWN(-1); // N.B. Majordomo V0.1 does not provide dedicated client commands + + private final byte[] data; + MdpClientCommand(final int value) { //watch for ints>255, will be truncated + this.data = new byte[] { (byte) (value & 0xFF) }; + } + + public boolean frameEquals(ZFrame frame) { + return Arrays.equals(data, frame.getData()); + } + + public byte[] getFrameData() { + return data; + } + + public ZFrame newFrame() { + return new ZFrame(data); + } + + public static MdpClientCommand getCommand(ZFrame frame) { + for (MdpClientCommand knownMdpCommand : MdpClientCommand.values()) { + if (knownMdpCommand.frameEquals(frame)) { + if (knownMdpCommand == C_UNKNOWN) { + continue; + } + return knownMdpCommand; + } + } + return C_UNKNOWN; + } + } + + public static class MdpMessage { + public final MdpSubProtocol protocol; + public final boolean isClient; + public final byte[] senderID; + public final String senderIdHex; + public final String senderName; + public final byte[][] payload; + + public MdpMessage(final byte[] senderID, final MdpSubProtocol protocol, final byte[]... payload) { + this.isClient = protocol == MdpSubProtocol.C_CLIENT; + this.senderID = senderID; + this.senderIdHex = strhex(senderID); + this.senderName = senderID == null ? null : new String(senderID, StandardCharsets.UTF_8); + this.protocol = protocol; + this.payload = payload; + } + + public byte[] getRbacFrame() { + if (hasRbackToken()) { + final byte[] rbacFrame = payload[payload.length - 1]; + return Arrays.copyOf(rbacFrame, rbacFrame.length); + } + return new byte[0]; + } + + public boolean hasRbackToken() { + return payload.length >= 2; + } + + @Override + public String toString() { + return "MdpMessage{isClient=" + isClient + ", senderID=" + ZData.toString(senderID) + ", payload=" + toString(payload) + '}'; + } + + protected static String toString(byte[][] byteValue) { + if (byteValue == null) { + return "(null)"; + } + if (byteValue.length == 0) { + return "[]"; + } + if (byteValue.length == 1) { + return "[" + ZData.toString(byteValue[0]) + "]"; + } + StringBuilder b = new StringBuilder(); + b.append('[').append(ZData.toString(byteValue[0])); + + for (int i = 1; i < byteValue.length; i++) { + b.append(", ").append(ZData.toString(byteValue[i])); + } + + b.append(']'); + return b.toString(); + } + } + + public static class MdpClientMessage extends MdpMessage { + public final MdpClientCommand command; + public final byte[] serviceNameBytes; // UTF-8 encoded service name + public final String serviceName; + public MdpClientMessage(final byte[] senderID, final MdpClientCommand clientCommand, final byte[] serviceNameBytes, final byte[]... clientMessages) { + super(senderID, MdpSubProtocol.C_CLIENT, clientMessages); + this.command = clientCommand; + this.serviceNameBytes = serviceNameBytes; + this.serviceName = new String(serviceNameBytes, StandardCharsets.UTF_8); + } + + @Override + public String toString() { + return "MdpClientMessage{senderID=" + ZData.toString(senderID) + ", serviceName='" + serviceName + "', payload=" + toString(payload) + '}'; + } + } + + public static class MdpWorkerMessage extends MdpMessage { + public final MdpWorkerCommand command; + public final byte[] serviceNameBytes; // UTF-8 encoded service name (optional - only for W_READY) + public final String serviceName; + public final byte[] clientSourceID; + public final String clientSourceName; + public MdpWorkerMessage(final byte[] senderID, final MdpWorkerCommand workerCommand, final byte[] serviceName, final byte[] clientSourceID, final byte[]... workerMessages) { + super(senderID, MdpSubProtocol.W_WORKER, workerMessages); + this.command = workerCommand; + this.serviceNameBytes = serviceName; + this.serviceName = serviceName == null ? null : new String(serviceName, StandardCharsets.UTF_8); + this.clientSourceID = clientSourceID; + this.clientSourceName = clientSourceID == null ? null : new String(clientSourceID, StandardCharsets.UTF_8); + } + + @Override + public String toString() { + return "MdpWorkerMessage{senderID=" + ZData.toString(senderID) + ", command=" + command + ", serviceName='" + serviceName + "', clientSourceID='" + ZData.toString(clientSourceID) + "', payload=" + toString(payload) + '}'; + } + } +} diff --git a/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoWorker.java b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoWorker.java new file mode 100644 index 00000000..70e6f7b5 --- /dev/null +++ b/concepts/src/main/java/io/opencmw/concepts/majordomo/MajordomoWorker.java @@ -0,0 +1,299 @@ +package io.opencmw.concepts.majordomo; + +import static io.opencmw.concepts.majordomo.MajordomoProtocol.*; +import static io.opencmw.concepts.majordomo.MajordomoProtocol.MdpWorkerCommand.*; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +import io.opencmw.rbac.RbacRole; +import io.opencmw.utils.SystemProperties; + +/** + * Majordomo Protocol Client API, Java version Implements the OpenCmwProtocol/Worker spec at + * http://rfc.zeromq.org/spec:7. + * + * default heart-beat time-out [ms] is set by system property: 'OpenCMW.heartBeat' // default: 2500 [ms] + * default heart-beat liveness is set by system property: 'OpenCMW.heartBeatLiveness' // [counts] 3-5 is reasonable + * N.B. heartbeat expires when last heartbeat message is more than HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms ago. + * this implies also, that worker must either return their message within 'HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms' or decouple their secondary handler interface into another thread. + * + */ +@SuppressWarnings("PMD.DoNotUseThreads") +public class MajordomoWorker extends Thread { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoWorker.class); + private static final int HEARTBEAT_LIVENESS = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeatLiveness", 3); // [counts] 3-5 is reasonable + private static final int HEARTBEAT_INTERVAL = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeat", 2500); // [ms] + private static final AtomicInteger WORKER_COUNTER = new AtomicInteger(); + + // --------------------------------------------------------------------- + protected final String uniqueID; + protected final ZContext ctx; + private final String brokerAddress; + private final String serviceName; + private final byte[] serviceBytes; + + private final AtomicBoolean run = new AtomicBoolean(true); // NOPMD + private final SortedSet> rbacRoles; + private ZMQ.Socket workerSocket; // Socket to broker + private long heartbeatAt; // When to send HEARTBEAT + private int liveness; // How many attempts left + private int reconnect = 2500; // Reconnect delay, msecs + private RequestHandler requestHandler; + private ZMQ.Poller poller; + + public MajordomoWorker(String brokerAddress, String serviceName, final RbacRole... rbacRoles) { + this(null, brokerAddress, serviceName, rbacRoles); + } + + public MajordomoWorker(ZContext ctx, String serviceName, final RbacRole... rbacRoles) { + this(ctx, "inproc://broker", serviceName, rbacRoles); + } + + protected MajordomoWorker(ZContext ctx, String brokerAddress, String serviceName, final RbacRole... rbacRoles) { + super(); + assert (brokerAddress != null); + assert (serviceName != null); + this.brokerAddress = brokerAddress; + this.serviceName = serviceName; + this.serviceBytes = serviceName.getBytes(StandardCharsets.UTF_8); + + // initialise RBAC role-based priority queues + this.rbacRoles = Collections.unmodifiableSortedSet(new TreeSet<>(Set.of(rbacRoles))); + + this.ctx = Objects.requireNonNullElseGet(ctx, ZContext::new); + if (ctx != null) { + this.setDaemon(true); + } + this.setName(MajordomoWorker.class.getSimpleName() + "#" + WORKER_COUNTER.getAndIncrement()); + this.uniqueID = this.serviceName + "-PID=" + ManagementFactory.getRuntimeMXBean().getName() + "-TID=" + this.getId(); + + this.setName(this.getClass().getSimpleName() + "(\"" + this.serviceName + "\")-" + uniqueID); + + LOGGER.atDebug().addArgument(serviceName).addArgument(uniqueID).log("created new service '{}' worker - uniqueID: {}"); + } + + public void destroy() { + ctx.destroy(); + } + + public int getHeartbeat() { + return HEARTBEAT_INTERVAL; + } + + public SortedSet> getRbacRoles() { + return rbacRoles; + } + + public int getReconnect() { + return reconnect; + } + + public RequestHandler getRequestHandler() { + return requestHandler; + } + + public String getServiceName() { + return serviceName; + } + + public String getUniqueID() { + return uniqueID; + } + + public MdpMessage handleRequestsFromBoker(final MdpWorkerMessage request) { + if (request == null) { + return null; + } + + switch (request.command) { + case W_REQUEST: + return processRequest(request, request.clientSourceID); + case W_HEARTBEAT: + // Do nothing for heartbeats + break; + case W_DISCONNECT: + reconnectToBroker(); + break; + case W_UNKNOWN: + default: + // N.B. not too verbose logging since we do not want that sloppy clients can bring down the broker through warning or info messages + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(uniqueID).addArgument(request).log("worer '{}' received invalid message: '{}'"); + } + break; + } + return null; + } + + public MdpMessage processRequest(final MdpMessage request, final byte[] clientSourceID) { + if (requestHandler != null) { + // de-serialise + // byte[] -> PropertyMap() (+ getObject(Class)) + try { + final byte[][] payload = requestHandler.handle(request.payload); + // serialise + return new MdpWorkerMessage(request.senderID, W_REPLY, serviceBytes, clientSourceID, payload); + } catch (Throwable e) { // NOPMD on purpose since we want to catch exceptions and courteously return this to the user + final StringWriter sw = new StringWriter(); + final PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + + //noinspection StringBufferReplaceableByString + final StringBuilder builder = new StringBuilder(); // NOPMD NOSONAR -- easier to read !?!? + builder.append(getClass().getName()) + .append(" caught exception in user-provided call-back function for service '") + .append(getServiceName()) + .append("'\nrequest msg: ") + .append(request) + .append("\nexception: ") + .append(sw.toString()); + final String exceptionMsg = builder.toString(); + return new MdpWorkerMessage(request.senderID, W_REPLY, serviceBytes, clientSourceID, exceptionMsg.getBytes(StandardCharsets.UTF_8)); + } + } + return null; + } + + public void registerHandler(final RequestHandler handler) { + this.requestHandler = handler; + } + + @Override + public void run() { + while (!Thread.currentThread().isInterrupted()) { + handleReceive(); + } + destroy(); + } + + public void setReconnect(int reconnect) { + this.reconnect = reconnect; + } + + @Override + public synchronized void start() { // NOPMD - need to be synchronised on class level due to super definition + run.set(true); + reconnectToBroker(); + super.start(); + } + + public void stopWorker() { + run.set(false); + } + + /** + * Send reply, if any, to broker and wait for next request. + */ + protected void handleReceive() { // NOPMD -- single serial function .. easier to read + while (run.get() && !Thread.currentThread().isInterrupted()) { + // Poll socket for a reply, with timeout + if (poller.poll(HEARTBEAT_INTERVAL) == -1) { + break; // Interrupted + } + + if (poller.pollin(0)) { + final MdpMessage msg = receiveMdpMessage(workerSocket); + if (msg == null) { + continue; + // break; // Interrupted + } + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(uniqueID).addArgument(msg).log("worker '{}' received new message from broker: '{}'"); + } + liveness = HEARTBEAT_LIVENESS; + // Don't try to handle errors, just assert noisily + assert msg.payload != null : "MdpWorkerMessage payload is null"; + if (!(msg instanceof MdpWorkerMessage)) { + assert false : "msg is not instance of MdpWorkerMessage"; + continue; + } + final MdpWorkerMessage workerMessage = (MdpWorkerMessage) msg; + + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(uniqueID).addArgument(workerMessage).log("worker '{}' received request: '{}'"); + } + + final MdpMessage reply = handleRequestsFromBoker(workerMessage); + + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(uniqueID).addArgument(reply).log("worker '{}' received reply: '{}'"); + } + + if (reply != null) { + sendWorkerMessage(workerSocket, W_REPLY, reply.senderID, workerMessage.clientSourceID, reply.payload); + } + + } else if (--liveness == 0) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(uniqueID).log("worker '{}' disconnected from broker - retrying"); + } + try { + //noinspection BusyWait + Thread.sleep(reconnect); // NOSONAR NOPMD -- need to wait until retry + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // Restore the interrupted status + break; + } + reconnectToBroker(); + } + + // Send HEARTBEAT if it's time + if (System.currentTimeMillis() > heartbeatAt) { + sendWorkerMessage(workerSocket, W_HEARTBEAT, null, null); + heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; + } + } + if (Thread.currentThread().isInterrupted()) { + LOGGER.atInfo().addArgument(uniqueID).log("worker '{}' interrupt received, killing worker"); + } + } + + /** + * Connect or reconnect to broker + */ + protected void reconnectToBroker() { + if (workerSocket != null) { + workerSocket.close(); + } + workerSocket = ctx.createSocket(SocketType.DEALER); + assert workerSocket != null : "worker socket is null"; + assert workerSocket.getSocketType() == SocketType.DEALER : "worker socket type is " + workerSocket.getSocketType() + " instead of " + SocketType.DEALER; + workerSocket.setHWM(0); + workerSocket.connect(brokerAddress); + LOGGER.atDebug().addArgument(uniqueID).addArgument(brokerAddress).log("worker '{}' connecting to broker at '{}'"); + + // Register service with broker + sendWorkerMessage(workerSocket, W_READY, null, serviceBytes, getUniqueID().getBytes(StandardCharsets.UTF_8)); + + if (poller != null) { + poller.unregister(workerSocket); + poller.close(); + } + poller = ctx.createPoller(1); + poller.register(workerSocket, ZMQ.Poller.POLLIN); + + // If liveness hits zero, queue is considered disconnected + liveness = HEARTBEAT_LIVENESS; + heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; + } + + public interface RequestHandler { + byte[][] handle(byte[][] payload) throws Throwable; + } +} diff --git a/concepts/src/main/java/module-info.java b/concepts/src/main/java/module-info.java new file mode 100644 index 00000000..1b31b8ec --- /dev/null +++ b/concepts/src/main/java/module-info.java @@ -0,0 +1,9 @@ +module io.opencmw.concepts { + requires org.slf4j; + requires disruptor; + requires jeromq; + requires java.management; + requires io.opencmw; + + exports io.opencmw.concepts.aggregate; +} \ No newline at end of file diff --git a/concepts/src/test/java/io/opencmw/concepts/BlockingQueueTests.java b/concepts/src/test/java/io/opencmw/concepts/BlockingQueueTests.java new file mode 100644 index 00000000..ddc3d1cd --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/BlockingQueueTests.java @@ -0,0 +1,95 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +public class BlockingQueueTests { // NOPMD NOSONAR -- nomen est omen + private static final int N_ITERATIONS = 100_000; + private final BlockingQueue outputQueue = new ArrayBlockingQueue<>(N_ITERATIONS); + private final BlockingQueue inputQueue = new ArrayBlockingQueue<>(N_ITERATIONS); + private final List workerList = new ArrayList<>(); + + public BlockingQueueTests(int nWorkers) { + for (int i = 0; i < nWorkers; i++) { + final int nWorker = i; + final Thread worker = new Thread() { + public void run() { + this.setName("worker#" + nWorker); + while (!this.isInterrupted()) { + final byte[] msg = outputQueue.poll(); + if (msg != null) { + inputQueue.offer(msg); + } + } + } + }; + + workerList.add(worker); + worker.start(); + } + } + + public void sendMessage(final byte[] msg) { + outputQueue.offer(msg); + } + + public byte[] receiveMessage() { + try { + return inputQueue.take(); + } catch (InterruptedException e) { + return null; + } + } + + public void stopWorker() { + workerList.forEach(Thread::interrupt); + } + + public static void main(String[] argv) { + for (int nThreads : new int[] { 1, 2, 4, 8, 10 }) { + final BlockingQueueTests test = new BlockingQueueTests(nThreads); + System.out.println("running: " + test.getClass().getName() + " with nThreads = " + nThreads); + + measure("nThreads=" + nThreads + " sync loop - simple", 3, () -> { + test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + System.out.println("received = " + (new String(msg, StandardCharsets.ISO_8859_1))); + }); + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " sync loop#" + i, N_ITERATIONS, () -> { + test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + } + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " async loop#" + i, N_ITERATIONS, () -> test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)), // + () -> { + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + } + test.stopWorker(); + } + } + + private static void measure(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.nanoTime(); + + for (Runnable run : runnable) { + for (int i = 0; i < nExec; i++) { + run.run(); + } + } + + final long stop = System.nanoTime(); + final double diff = (stop - start) * 1e-9; + System.out.printf("%-40s: %10d calls/second\n", topic, diff > 0 ? (int) (nExec / diff) : -1); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/DisruptorTests.java b/concepts/src/test/java/io/opencmw/concepts/DisruptorTests.java new file mode 100644 index 00000000..ac385f4c --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/DisruptorTests.java @@ -0,0 +1,115 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; + +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; +import com.lmax.disruptor.util.DaemonThreadFactory; +import com.lmax.disruptor.util.Util; + +public class DisruptorTests { // NOPMD NOSONAR -- nomen est omen + // https://github.com/LMAX-Exchange/disruptor/wiki/Getting-Started + private static final int N_ITERATIONS = 100_000; + private static final int BUFFER_SIZE = Util.ceilingNextPowerOfTwo(N_ITERATIONS); // specify the size of the ring buffer, must be power of 2. + private final Disruptor disruptorOut; + private final RingBuffer outputBuffer; + private final Disruptor disruptorIn; + private final RingBuffer inputBuffer; + private long readPosition = 0; + + public DisruptorTests(int nWorkers) { + disruptorOut = new Disruptor<>(ByteArrayEvent::new, BUFFER_SIZE, DaemonThreadFactory.INSTANCE, ProducerType.SINGLE, new BlockingWaitStrategy()); + outputBuffer = disruptorOut.getRingBuffer(); + disruptorIn = new Disruptor<>(ByteArrayEvent::new, BUFFER_SIZE, DaemonThreadFactory.INSTANCE, nWorkers <= 1 ? ProducerType.SINGLE : ProducerType.MULTI, new BlockingWaitStrategy()); + inputBuffer = disruptorIn.getRingBuffer(); + + // Connect the parallel handler + for (int i = 0; i < nWorkers; i++) { + final int threadWorkerID = i; + disruptorOut.handleEventsWith((inputEvent, sequence, endOfBatch) -> { + if (sequence % nWorkers != threadWorkerID) { + return; + } + inputBuffer.publishEvent((returnEvent, returnSequence, returnBuffer) -> returnEvent.array = inputEvent.array); + }); + } + // Start the Disruptor, starts all threads running + disruptorOut.start(); + disruptorIn.start(); + } + + public void sendMessage(final byte[] msg) { + outputBuffer.publishEvent((event, sequence, buffer) -> event.array = msg); + } + + public byte[] receiveMessage() { + //System.err.println("inputBuffer.getCursor() = " + inputBuffer.getCursor()); + long cursor; + //noinspection StatementWithEmptyBody + while ((cursor = inputBuffer.getCursor()) < 0 || readPosition > cursor) { // NOPMD NOSONAR -- busy loop + // empty block on purpose - busy loop optimises latency + } + if (readPosition <= cursor) { + final ByteArrayEvent value = inputBuffer.get(readPosition); + readPosition++; + return value.array; + } else { + return null; + } + } + + public void stopWorker() { + disruptorOut.shutdown(); + disruptorIn.shutdown(); + } + + public static void main(String[] argv) { + for (int nThreads : new int[] { 1, 2, 4, 8, 10 }) { + final DisruptorTests test = new DisruptorTests(nThreads); + System.out.println("running: " + test.getClass().getName() + " with nThreads = " + nThreads); + + measure("nThreads=" + nThreads + " sync loop - simple", 3, () -> { + test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " sync loop#" + i, N_ITERATIONS, () -> { + test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + } + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " async loop#" + i, N_ITERATIONS, () -> test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)), // + () -> { + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + } + test.stopWorker(); + } + } + + private static void measure(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.nanoTime(); + + for (Runnable run : runnable) { + for (int i = 0; i < nExec; i++) { + run.run(); + } + } + + final long stop = System.nanoTime(); + final double diff = (stop - start) * 1e-9; + System.out.printf("%-40s: %10d calls/second\n", topic, diff > 0 ? (int) (nExec / diff) : -1); + } + + private static class ByteArrayEvent { + public byte[] array; + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/FutureTests.java b/concepts/src/test/java/io/opencmw/concepts/FutureTests.java new file mode 100644 index 00000000..9be71558 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/FutureTests.java @@ -0,0 +1,216 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.jetbrains.annotations.NotNull; + +public class FutureTests { // NOPMD NOSONAR -- nomen est omen + private static final int N_ITERATIONS = 100_000; + private final BlockingQueue> outputQueue = new ArrayBlockingQueue<>(N_ITERATIONS); + private final List workerList = new ArrayList<>(); + + public FutureTests(int nWorkers) { + for (int i = 0; i < nWorkers; i++) { + final int nWorker = i; + final Thread worker = new Thread() { + public void run() { + this.setName("worker#" + nWorker); + try { + while (!this.isInterrupted()) { + final CustomFuture msgFuture; + if (outputQueue.isEmpty()) { + msgFuture = outputQueue.take(); + } else { + msgFuture = outputQueue.poll(); + } + if (msgFuture == null) { + continue; + } + msgFuture.running.set(true); + if (msgFuture.payload != null) { + msgFuture.setReply(msgFuture.payload); + continue; + } + msgFuture.cancelled.set(true); + } + } catch (InterruptedException e) { + if (!outputQueue.isEmpty()) { + e.printStackTrace(); + } + } + } + }; + + workerList.add(worker); + worker.start(); + } + } + + public Future sendMessage(final byte[] msg) { + CustomFuture msgFuture = new CustomFuture<>(msg); + outputQueue.offer(msgFuture); + return msgFuture; + } + + public void stopWorker() { + workerList.forEach(Thread::interrupt); + } + + public static void main(String[] argv) { + for (int nThreads : new int[] { 1, 2, 4, 8, 10 }) { + final FutureTests test = new FutureTests(nThreads); + System.out.println("running: " + test.getClass().getName() + " with nThreads = " + nThreads); + + measure("nThreads=" + nThreads + " sync loop - simple", 3, () -> { + final Future reply = test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + try { + final byte[] msg = reply.get(); + assert msg != null : "message must be non-null"; + System.out.println("received = " + (new String(msg, StandardCharsets.ISO_8859_1))); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + }); + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " sync loop#" + i, N_ITERATIONS, () -> { + final Future reply = test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + try { + final byte[] msg = reply.get(); + assert msg != null : "message must be non-null"; + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + }); + } + + for (int i = 0; i < 3; i++) { + measureAsync("nThreads=" + nThreads + " async loop#" + i, N_ITERATIONS, + () -> { + final List> replies = new ArrayList<>(N_ITERATIONS); + for (int k = 0; k < N_ITERATIONS; k++) { + replies.add(test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1))); + } + assert replies.size() == N_ITERATIONS : "did not receive sufficient events"; + replies.forEach(reply -> { + try { + final byte[] msg = reply.get(); + assert msg != null : "message must be non-null"; + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + }); + }); + } + test.stopWorker(); + } + } + + private static void measure(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.nanoTime(); + + for (Runnable run : runnable) { + for (int i = 0; i < nExec; i++) { + run.run(); + } + } + + final long stop = System.nanoTime(); + final double diff = (stop - start) * 1e-9; + System.out.printf("%-40s: %10d calls/second\n", topic, diff > 0 ? (int) (nExec / diff) : -1); + } + + private static void measureAsync(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.nanoTime(); + + for (Runnable run : runnable) { + run.run(); + } + + final long stop = System.nanoTime(); + final double diff = (stop - start) * 1e-9; + System.out.printf("%-40s: %10d calls/second\n", topic, diff > 0 ? (int) (nExec / diff) : -1); + } + + private static class CustomFuture implements Future { + private final Lock lock = new ReentrantLock(); + private final Condition processorNotifyCondition = lock.newCondition(); + private final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean requestCancel = new AtomicBoolean(false); + private final AtomicBoolean cancelled = new AtomicBoolean(false); + private final T payload; + private T reply = null; + + private CustomFuture(final T input) { + this.payload = input; + } + + @Override + public boolean cancel(final boolean mayInterruptIfRunning) { + if (!running.get()) { + cancelled.set(true); + return !requestCancel.getAndSet(true); + } + return false; + } + + @Override + public T get() throws InterruptedException { + return get(0, TimeUnit.NANOSECONDS); + } + + @Override + public T get(final long timeout, @NotNull final TimeUnit unit) throws InterruptedException { + if (isDone()) { + return reply; + } + lock.lock(); + try { + while (!isDone()) { + //noinspection ResultOfMethodCallIgnored + processorNotifyCondition.await(timeout, TimeUnit.NANOSECONDS); + } + } finally { + lock.unlock(); + } + return reply; + } + + @Override + public boolean isCancelled() { + return cancelled.get(); + } + + @Override + public boolean isDone() { + return (reply != null && !running.get()) || cancelled.get(); + } + + public void setReply(final T newValue) { + if (running.getAndSet(false)) { + this.reply = newValue; + } + notifyListener(); + } + + private void notifyListener() { + lock.lock(); + try { + processorNotifyCondition.signalAll(); + } finally { + lock.unlock(); + } + } + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/ManyVsLargeFrameEvaluation.java b/concepts/src/test/java/io/opencmw/concepts/ManyVsLargeFrameEvaluation.java new file mode 100644 index 00000000..545ba8c1 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/ManyVsLargeFrameEvaluation.java @@ -0,0 +1,281 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMQ.Socket; +import org.zeromq.ZMQException; +import org.zeromq.ZMsg; + +/** + * Quick performance evaluation to see the impact of single large w.r.t. many small frames. + */ +@SuppressWarnings({ "PMD.DoNotUseThreads", "PMD.AvoidInstantiatingObjectsInLoops" }) +public class ManyVsLargeFrameEvaluation { + private static final Logger LOGGER = LoggerFactory.getLogger(ManyVsLargeFrameEvaluation.class); + private static final AtomicBoolean RUN = new AtomicBoolean(true); + private static final byte[] CLIENT_ID = "C".getBytes(StandardCharsets.UTF_8); // client name + private static final byte[] WORKER_ID = "W".getBytes(StandardCharsets.UTF_8); // worker-service name + public static final char TAG_INTERNAL = 'I'; + public static final char TAG_EXTERNAL = 'E'; + public static final String TAG_EXTERNAL_STRING = "E"; + public static final String TAG_EXTERNAL_INTERNAL = "I"; + public static final String ENDPOINT_ROUTER = "tcp://localhost:5555"; + public static final String ENDPOINT_TCP = "tcp://localhost:5556"; + public static final String ENDPOINT_PUBSUB = "tcp://localhost:5557"; + private static int sampleSize = 100_000; + private static final int N_BUFFER_SIZE = 8; + private static final int N_FRAMES = 10; + public static final byte[] smallMessage = new byte[N_BUFFER_SIZE * N_FRAMES]; // NOPMD - volatile on purpose + public static final byte[] largeMessage = new byte[N_BUFFER_SIZE]; // NOPMD - volatile on purpose + + private static final int N_LOOPS = 5; + + private ManyVsLargeFrameEvaluation() { + // utility class + } + + public static void main(String[] args) { + if (args.length == 1) { + sampleSize = Integer.parseInt(args[0]); + } + Thread brokerThread = new Thread(new Broker()); + Thread workerThread = new Thread(new RoundTripAndNotifyEvaluation.Worker()); + brokerThread.start(); + workerThread.start(); + + Thread clientThread = new Thread(new Client()); + clientThread.start(); + + try { + clientThread.join(); + RUN.set(false); + workerThread.interrupt(); + brokerThread.interrupt(); + + // wait for threads to finish + workerThread.join(); + brokerThread.join(); + } catch (InterruptedException e) { + // finishes tests + assert false : "should not reach here if properly executed"; + } + + LOGGER.atDebug().log("finished tests"); + } + + private static void measure(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.currentTimeMillis(); + + for (Runnable run : runnable) { + for (int i = 0; i < nExec; i++) { + run.run(); + } + } + + final long stop = System.currentTimeMillis(); + LOGGER.atDebug().addArgument(String.format("%-40s: %10d calls/second", topic, (1000L * nExec) / (stop - start))).log("{}"); + } + + private static class Broker implements Runnable { + private static final int TIMEOUT = 1000; + + @Override + public void run() { // NOPMD single-loop broker ... simplifies reading + try (ZContext ctx = new ZContext(); + Socket tcpFrontend = ctx.createSocket(SocketType.ROUTER); + Socket tcpBackend = ctx.createSocket(SocketType.ROUTER); + Socket inprocBackend = ctx.createSocket(SocketType.ROUTER); + ZMQ.Poller items = ctx.createPoller(3)) { + tcpFrontend.setHWM(0); + tcpBackend.setHWM(0); + inprocBackend.setHWM(0); + tcpFrontend.bind(ENDPOINT_ROUTER); + tcpBackend.bind(ENDPOINT_TCP); + inprocBackend.bind("inproc://broker"); + + Thread internalWorkerThread = new Thread(new InternalWorker(ctx)); + internalWorkerThread.setDaemon(true); + internalWorkerThread.start(); + items.register(tcpFrontend, ZMQ.Poller.POLLIN); + items.register(tcpBackend, ZMQ.Poller.POLLIN); + items.register(inprocBackend, ZMQ.Poller.POLLIN); + + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + if (items.poll(TIMEOUT) == -1) { + break; // Interrupted + } + + if (items.pollin(0)) { + ZMsg msg = ZMsg.recvMsg(tcpFrontend); + if (msg == null) { + break; // Interrupted + } + + final ZFrame address = msg.pop(); + if (address.getData()[0] != CLIENT_ID[0]) { + address.destroy(); + break; + } + final ZFrame internal = msg.pop(); + if (TAG_EXTERNAL == internal.getData()[0]) { + msg.addFirst(new ZFrame(WORKER_ID)); // NOPMD - necessary to allocate inside loop + msg.send(tcpBackend); + } else if (TAG_INTERNAL == internal.getData()[0]) { + msg.addFirst(new ZFrame(WORKER_ID)); + msg.send(inprocBackend); + } + address.destroy(); + } + + if (items.pollin(1)) { + ZMsg msg = ZMsg.recvMsg(tcpBackend); + if (msg == null) { + break; // Interrupted + } + ZFrame address = msg.pop(); + + if (address.getData()[0] == WORKER_ID[0]) { + msg.addFirst(new ZFrame(CLIENT_ID)); + } + msg.send(tcpFrontend); + address.destroy(); + } + + if (items.pollin(2)) { + final ZMsg msg = ZMsg.recvMsg(inprocBackend); + if (msg == null) { + break; // Interrupted + } + final ZFrame address = msg.pop(); + + if (address.getData()[0] == WORKER_ID[0]) { + msg.addFirst(new ZFrame(CLIENT_ID)); + } + address.destroy(); + msg.send(tcpFrontend); + } + } + + internalWorkerThread.interrupt(); + if (!internalWorkerThread.isInterrupted()) { + internalWorkerThread.join(); + } + } catch (InterruptedException | IllegalStateException e) { + // terminated broker via interrupt + } + } + + private static class InternalWorker implements Runnable { + private final ZContext ctx; + + private InternalWorker(ZContext ctx) { + this.ctx = ctx; + } + + @Override + public void run() { + try (Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(WORKER_ID); + worker.connect("inproc://broker"); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + msg.send(worker); + } + } catch (ZMQException e) { + // terminate internal worker + } + } + } + } + + private static class Client implements Runnable { + @Override + public void run() { // NOPMD -- complexity + try (ZContext ctx = new ZContext(); + Socket client = ctx.createSocket(SocketType.DEALER); + Socket subClient = ctx.createSocket(SocketType.SUB)) { + client.setHWM(0); + client.setIdentity(CLIENT_ID); + client.connect(ENDPOINT_ROUTER); + subClient.setHWM(0); + subClient.connect(ENDPOINT_PUBSUB); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); + + LOGGER.atDebug().log("Setting up test"); + + for (int l = 0; l < N_LOOPS; l++) { + for (final boolean external : new boolean[] { true, false }) { + final String inOut = external ? "TCP " : "InProc"; + measure(" Synchronous round-trip test (" + inOut + ", large frames)", sampleSize, () -> { + ZMsg req = new ZMsg(); + req.addString(external ? TAG_EXTERNAL_STRING : TAG_EXTERNAL_INTERNAL); + for (int i = 0; i < N_FRAMES; i++) { + req.add(smallMessage); + } + req.send(client); + ZMsg.recvMsg(client).destroy(); + }); + + measure("Asynchronous round-trip test (" + inOut + ", large frames)", sampleSize, () -> { + // send messages + ZMsg req = new ZMsg(); + req.addString(external? TAG_EXTERNAL_STRING : TAG_EXTERNAL_INTERNAL); + for (int i = 0; i < N_FRAMES; i++) { + req.add(smallMessage); + } + req.send(client); }, () -> { + // receive messages + ZMsg.recvMsg(client).destroy(); }); + + measure(" Synchronous round-trip test (" + inOut + ", many frames)", sampleSize, () -> { + ZMsg req = new ZMsg(); + req.addString(external ? TAG_EXTERNAL_STRING : TAG_EXTERNAL_INTERNAL); + req.add(largeMessage); + req.send(client); + ZMsg.recvMsg(client).destroy(); + }); + + measure("Asynchronous round-trip test (" + inOut + ", many frames)", sampleSize, () -> { + // send messages + ZMsg req = new ZMsg(); + req.addString(external? TAG_EXTERNAL_STRING : TAG_EXTERNAL_INTERNAL); + req.add(largeMessage); + req.send(client); }, () -> { + // receive messages + ZMsg.recvMsg(client).destroy(); }); + } + } + } catch (ZMQException e) { + LOGGER.atDebug().log("terminate client"); + } + } + } + + private static class Worker implements Runnable { + @Override + public void run() { + try (ZContext ctx = new ZContext(); + Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(WORKER_ID); + worker.connect(ENDPOINT_TCP); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + msg.send(worker); + } + } catch (ZMQException e) { + // terminate worker + } + } + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/PushPullTests.java b/concepts/src/test/java/io/opencmw/concepts/PushPullTests.java new file mode 100644 index 00000000..63161e76 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/PushPullTests.java @@ -0,0 +1,121 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMQException; + +public class PushPullTests { // NOPMD NOSONAR -- nomen est omen + private static final int N_ITERATIONS = 100_000; + private final List workerList = new ArrayList<>(); + private final ZContext ctx = new ZContext(); + private final ZMQ.Socket sendSocket; + private final ZMQ.Socket receiveSocket; + + public PushPullTests(int nWorkers) { + sendSocket = ctx.createSocket(SocketType.PUSH); + sendSocket.setHWM(0); + sendSocket.bind("inproc://broker_push"); + receiveSocket = ctx.createSocket(SocketType.PULL); + receiveSocket.setHWM(0); + receiveSocket.bind("inproc://broker_pull"); + receiveSocket.setReceiveTimeOut(-1); + + for (int i = 0; i < nWorkers; i++) { + final int nWorker = i; + final Thread worker = new Thread() { + private final ZMQ.Socket sendSocket = ctx.createSocket(SocketType.PUSH); + private final ZMQ.Socket receiveSocket = ctx.createSocket(SocketType.PULL); + public void run() { + this.setName("worker#" + nWorker); + receiveSocket.connect("inproc://broker_push"); + receiveSocket.setHWM(0); + receiveSocket.setReceiveTimeOut(-1); + sendSocket.connect("inproc://broker_pull"); + try { + while (!this.isInterrupted()) { + final byte[] msg = receiveSocket.recv(0); + if (msg != null) { + sendSocket.send(msg); + } + } + } catch (ZMQException e) { + // process should abort + receiveSocket.close(); + sendSocket.close(); + } + } + }; + + workerList.add(worker); + worker.start(); + } + } + + public void sendMessage(final byte[] msg) { + sendSocket.send(msg); + } + + public byte[] receiveMessage() { + return receiveSocket.recv(); + } + + public void stopWorker() { + try { + Thread.sleep(1000); // NOSONAR + } catch (InterruptedException e) { + // do nothing + } + workerList.forEach(Thread::interrupt); + } + + public static void main(String[] argv) { + for (int nThreads : new int[] { 1, 2, 4, 8, 10 }) { + final PushPullTests test = new PushPullTests(nThreads); + System.out.println("running: " + test.getClass().getName() + " with nThreads = " + nThreads); + + measure("nThreads=" + nThreads + " sync loop - simple", 3, () -> { + test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + System.out.println("received = " + (new String(msg, StandardCharsets.ISO_8859_1))); + }); + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " sync loop#" + i, N_ITERATIONS, () -> { + test.sendMessage("testString".getBytes(StandardCharsets.ISO_8859_1)); + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + } + + for (int i = 0; i < 3; i++) { + measure("nThreads=" + nThreads + " async loop#" + i, N_ITERATIONS, () -> test.sendMessage("testStringA".getBytes(StandardCharsets.ISO_8859_1)), // + () -> { + final byte[] msg = test.receiveMessage(); + assert msg != null : "message must be non-null"; + }); + } + + test.stopWorker(); + } + } + + private static void measure(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.nanoTime(); + + for (Runnable run : runnable) { + for (int i = 0; i < nExec; i++) { + run.run(); + } + } + + final long stop = System.nanoTime(); + final double diff = (stop - start) * 1e-9; + System.out.printf("%-40s: %10d calls/second\n", topic, diff > 0 ? (int) (nExec / diff) : -1); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/RestBehindRouterEvaluation.java b/concepts/src/test/java/io/opencmw/concepts/RestBehindRouterEvaluation.java new file mode 100644 index 00000000..1af8da58 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/RestBehindRouterEvaluation.java @@ -0,0 +1,210 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Timer; +import java.util.TimerTask; + +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ.Poller; +import org.zeromq.ZMQ.Socket; + +public class RestBehindRouterEvaluation { // NOPMD -- nomen est omen + private static final byte[] ZERO_MQ_HEADER = { -1, 0, 0, 0, 0, 0, 0, 0, 7, 127 }; + + private RestBehindRouterEvaluation() { + // utility class + } + + public static void main(final String[] argv) { + try (ZContext context = new ZContext(); + Socket tcpProxy = context.createSocket(SocketType.ROUTER); + Socket router = context.createSocket(SocketType.ROUTER); + Socket stream = context.createSocket(SocketType.STREAM); + Poller poller = context.createPoller(2)) { + tcpProxy.setRouterRaw(true); + if (!tcpProxy.bind("tcp://*:8080")) { + throw new IllegalStateException("could not bind socket"); + } + if (!router.bind("tcp://*:8081")) { + throw new IllegalStateException("could not bind socket"); + } + if (!stream.bind("tcp://*:8082")) { + throw new IllegalStateException("could not bind socket"); + } + poller.register(tcpProxy, Poller.POLLIN); + poller.register(stream, Poller.POLLIN); + + final TimerTask timerTask = new TimerTask() { + @Override + public void run() { + try (Socket dealer = context.createSocket(SocketType.DEALER)) { + dealer.setIdentity("dealer".getBytes(zmq.ZMQ.CHARSET)); + + System.err.println("clients sends request"); + dealer.connect("tcp://localhost:8080"); + dealer.send("Hello World"); + } + } + }; + new Timer().schedule(timerTask, 2000); + + while (!Thread.interrupted()) { + if (poller.poll(1000) == -1) { + break; // Interrupted + } + + if (poller.pollin(0)) { + handleRouterSocket(tcpProxy); + } + if (poller.pollin(1)) { + handleStreamHttpSocket(stream, null); + } + } + } + } + + private static long bytesToLong(byte[] bytes) { + long value = 0; + for (final byte aByte : bytes) { + value = (value << 8) + (aByte & 0xff); + } + return value; + } + + private static ZFrame getConnectionID(final Socket socket) { + // TODO: add further safe-guards if called for a socket with no data pending + + // Get [id, ] message on client connection. + final ZFrame handle = ZFrame.recvFrame(socket); + if (handle == null || bytesToLong(handle.getData()) == 0) { + return null; + } + + System.err.println("received ID = " + handle.toString()); // Professional Logging(TM) + if (!handle.hasMore()) { + // Close erroneous connection to browser + handle.send(socket, ZFrame.MORE | ZFrame.REUSE); + socket.send((byte[]) null, 0); + return null; + } + + // receive empty payload. + final ZFrame emptyFrame = ZFrame.recvFrame(socket); + if (!emptyFrame.hasMore() || emptyFrame.size() == 0) { + // received null frame + //System.err.println("nothing received"); + return handle; + } + if (emptyFrame.hasMore() || emptyFrame.size() != 0) { + System.err.println("did receive more " + emptyFrame); + return null; + } + + return handle; + } + + private static ZFrame getRequest(final Socket socket) { + // TODO: add further safe-guards if called for a socket with no data pending + + // Get [id, ] message on client connection. + final ZFrame handle = ZFrame.recvFrame(socket); + if (handle == null || bytesToLong(handle.getData()) == 0) { + return null; + } + + System.err.println("received Request ID = " + handle.toString() + " - more = " + handle.hasMore()); // Professional Logging(TM) + if (!handle.hasMore()) { + // Close erroneous connection to browser + handle.send(socket, ZFrame.MORE | ZFrame.REUSE); + socket.send((byte[]) null, 0); + return null; + } + + // receive request + return ZFrame.recvFrame(socket); + } + + private static void handleRouterSocket(final Socket router) { + System.err.println("### called handleRouterSocket"); + // Get [id, ] message on client connection. + ZFrame handle = getConnectionID(router); + if (handle == null) { + // did not receive proper [ID, null msg] frames + return; + } + + final ZFrame request = getRequest(router); + if (request == null) { + return; + } + if (Arrays.equals(ZERO_MQ_HEADER, request.getData())) { + router.sendMore(handle.getData()); + router.send(ZERO_MQ_HEADER); + System.err.println("received ZeroMQ message more = " + request.hasMore()); + + } else { + System.err.println("received other (HTTP) message"); + System.err.println("received request = " + request + " more? " + request.hasMore()); + return; + } + + // handleStreamHttpSocket(router, handle); + // received router request + while (request.hasMore()) { + // receive message + final byte[] message = router.recv(0); + final boolean more = router.hasReceiveMore(); + System.err.println("router msg (" + (more ? "more" : "all ") + "): " + Arrays.toString(message) + "\n - string: '" + new String(message) + "'"); // NOPMD + + //handleStreamHttpSocket(router); + // Broker it -- throws an exception (too naive implementation?) + //stream.send(message, more ? ZMQ.SNDMORE : 0); + if (!more) { + break; + } + } + } + + private static void handleStreamHttpSocket(Socket httpSocket, ZFrame handlerExt) { + // Get [id, ] message on client connection. + ZFrame handler = handlerExt; + if (handler == null && (handler = getConnectionID(httpSocket)) == null) { // NOPMD + // did not receive proper [ID, null msg] frames + return; + } + + // Get [id, playload] message. + final ZFrame clientRequest = ZFrame.recvFrame(httpSocket); + if (clientRequest == null || bytesToLong(clientRequest.getData()) == 0) { + return; + } + if (!Arrays.equals(handler.getData(), clientRequest.getData())) { + // header ID mismatch + return; + } + if (!clientRequest.hasMore()) { + // Close erroneous connection to browser + clientRequest.send(httpSocket, ZFrame.MORE | ZFrame.REUSE); + httpSocket.send((byte[]) null, 0); + return; + } + + // receive playload message. + ZFrame request = ZFrame.recvFrame(httpSocket); + String header = new String(request.getData(), 0, request.size(), + StandardCharsets.UTF_8); + System.err.println("received client request header : '" + header); // Professional Logging(TM) + + // Send Hello World response + final String uri = (header.length() == 0) ? "null" : header.split("\n")[0]; + clientRequest.send(httpSocket, ZFrame.MORE | ZFrame.REUSE); + httpSocket.send("HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nHello, World!\nyou requested URI: " + uri); + + // Close connection to browser -- normally exit + clientRequest.send(httpSocket, ZFrame.MORE | ZFrame.REUSE); + httpSocket.send((byte[]) null, 0); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/RoundTripAndNotifyEvaluation.java b/concepts/src/test/java/io/opencmw/concepts/RoundTripAndNotifyEvaluation.java new file mode 100644 index 00000000..e30514bf --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/RoundTripAndNotifyEvaluation.java @@ -0,0 +1,449 @@ +package io.opencmw.concepts; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMQ.Socket; +import org.zeromq.ZMQException; +import org.zeromq.ZMsg; + +/** + * Quick Router-Dealer Round-trip demonstrator. + * Broker, Worker and Client are mocked as separate threads. + * + * Example output: + * Setting up test + * Synchronous round-trip test (TCP) : 3838 calls/second + * Synchronous round-trip test (InProc) : 12224 calls/second + * Asynchronous round-trip test (TCP) : 33444 calls/second + * Asynchronous round-trip test (InProc) : 35587 calls/second + * Subscription (SUB) test : 632911 calls/second + * Subscription (DEALER) test (TCP) : 43821 calls/second + * Subscription (DEALER) test (InProc) : 49900 calls/second + * Subscription (direct DEALER) test : 1351351 calls/second + * finished tests + * + * N.B. for >200000 calls/second the code seems to depend largely on the broker/parameters + * (ie. JIT, whether services are identified by single characters etc.) + */ +@SuppressWarnings({ "PMD.DoNotUseThreads", "PMD.AvoidInstantiatingObjectsInLoops" }) +class RoundTripAndNotifyEvaluation { + private static final Logger LOGGER = LoggerFactory.getLogger(RoundTripAndNotifyEvaluation.class); + // private static final String SUB_TOPIC = "x"; + private static final String SUB_TOPIC = "/?# - a very long topic to test the dependence of pub/sub pairs on topic lengths"; + private static final byte[] SUB_DATA = "D".getBytes(StandardCharsets.UTF_8); // custom minimal data + private static final byte[] CLIENT_ID = "C".getBytes(StandardCharsets.UTF_8); // client name + private static final byte[] WORKER_ID = "W".getBytes(StandardCharsets.UTF_8); // worker-service name + private static final byte[] PUBLISH_ID = "P".getBytes(StandardCharsets.UTF_8); // publish-service name + private static final byte[] SUBSCRIBER_ID = "S".getBytes(StandardCharsets.UTF_8); // subscriber name + public static final char TAG_INTERNAL = 'I'; + public static final char TAG_EXTERNAL = 'E'; + public static final String TAG_EXTERNAL_STRING = "E"; + public static final String TAG_EXTERNAL_INTERNAL = "I"; + public static final String START = "start"; + public static final String DUMMY_DATA = "hello"; + public static final String ENDPOINT_ROUTER = "tcp://localhost:5555"; + public static final String ENDPOINT_TCP = "tcp://localhost:5556"; + public static final String ENDPOINT_PUBSUB = "tcp://localhost:5557"; + private static final AtomicBoolean RUN = new AtomicBoolean(true); + private static int sampleSize = 10_000; + private static int sampleSizePub = 100_000; + + private RoundTripAndNotifyEvaluation() { + // utility class + } + + public static void main(String[] args) { + if (args.length == 1) { + sampleSize = Integer.parseInt(args[0]); + sampleSizePub = 10 * sampleSize; + } + Thread brokerThread = new Thread(new Broker()); + Thread workerThread = new Thread(new Worker()); + Thread publishThread = new Thread(new PublishWorker()); + Thread pubDealerThread = new Thread(new PublishDealerWorker()); + Thread directDealerThread = new Thread(new PublishDirectDealerWorker()); + + brokerThread.start(); + workerThread.start(); + publishThread.start(); + pubDealerThread.start(); + directDealerThread.start(); + + Thread clientThread = new Thread(new Client()); + clientThread.start(); + + try { + clientThread.join(); + RUN.set(false); + workerThread.interrupt(); + brokerThread.interrupt(); + publishThread.interrupt(); + pubDealerThread.interrupt(); + directDealerThread.interrupt(); + + // wait for threads to finish + workerThread.join(); + publishThread.join(); + pubDealerThread.join(); + directDealerThread.join(); + brokerThread.join(); + } catch (InterruptedException e) { + // finishes tests + assert false : "should not reach here if properly executed"; + } + + LOGGER.atDebug().log("finished tests"); + } + + private static void measure(final String topic, final int nExec, final Runnable... runnable) { + final long start = System.currentTimeMillis(); + + for (Runnable run : runnable) { + for (int i = 0; i < nExec; i++) { + run.run(); + } + } + + final long stop = System.currentTimeMillis(); + LOGGER.atDebug().addArgument(String.format("%-40s: %10d calls/second", topic, (1000L * nExec) / (stop - start))).log("{}"); + } + + private static class Broker implements Runnable { + private static final int TIMEOUT = 1000; + @Override + public void run() { // NOPMD single-loop broker ... simplifies reading + try (ZContext ctx = new ZContext(); + Socket tcpFrontend = ctx.createSocket(SocketType.ROUTER); + Socket tcpBackend = ctx.createSocket(SocketType.ROUTER); + Socket inprocBackend = ctx.createSocket(SocketType.ROUTER); + ZMQ.Poller items = ctx.createPoller(3)) { + tcpFrontend.setHWM(0); + tcpBackend.setHWM(0); + inprocBackend.setHWM(0); + final boolean a = tcpFrontend.bind(ENDPOINT_ROUTER); + tcpBackend.bind(ENDPOINT_TCP); + inprocBackend.bind("inproc://broker"); + items.register(tcpFrontend, ZMQ.Poller.POLLIN); + items.register(tcpBackend, ZMQ.Poller.POLLIN); + items.register(inprocBackend, ZMQ.Poller.POLLIN); + + Thread internalWorkerThread = new Thread(new InternalWorker(ctx)); + internalWorkerThread.setDaemon(true); + internalWorkerThread.start(); + Thread internalPublishDealerWorkerThread = new Thread(new InternalPublishDealerWorker(ctx)); + internalPublishDealerWorkerThread.setDaemon(true); + internalPublishDealerWorkerThread.start(); + + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + if (items.poll(TIMEOUT) == -1) { + break; // Interrupted + } + + if (items.pollin(0)) { + final ZMsg msg = ZMsg.recvMsg(tcpFrontend); + if (msg == null) { + break; // Interrupted + } + final ZFrame address = msg.pop(); + final ZFrame internal = msg.pop(); + if (address.getData()[0] == CLIENT_ID[0]) { + if (TAG_EXTERNAL == internal.getData()[0]) { + msg.addFirst(new ZFrame(WORKER_ID)); + msg.send(tcpBackend); + } else if (TAG_INTERNAL == internal.getData()[0]) { + msg.addFirst(new ZFrame(WORKER_ID)); + msg.send(inprocBackend); + } + } else { + if (TAG_EXTERNAL == internal.getData()[0]) { + msg.addFirst(new ZFrame(PUBLISH_ID)); + msg.send(tcpBackend); + } else if (TAG_INTERNAL == internal.getData()[0]) { + msg.addFirst(new ZFrame(PUBLISH_ID)); + msg.send(inprocBackend); + } + } + address.destroy(); + } + + if (items.pollin(1)) { + ZMsg msg = ZMsg.recvMsg(tcpBackend); + if (msg == null) { + break; // Interrupted + } + ZFrame address = msg.pop(); + + if (address.getData()[0] == WORKER_ID[0]) { + msg.addFirst(new ZFrame(CLIENT_ID)); + } else { + msg.addFirst(new ZFrame(SUBSCRIBER_ID)); + } + msg.send(tcpFrontend); + address.destroy(); + } + + if (items.pollin(2)) { + final ZMsg msg = ZMsg.recvMsg(inprocBackend); + if (msg == null) { + break; // Interrupted + } + ZFrame address = msg.pop(); + + if (address.getData()[0] == WORKER_ID[0]) { + msg.addFirst(new ZFrame(CLIENT_ID)); + } else { + msg.addFirst(new ZFrame(SUBSCRIBER_ID)); + } + address.destroy(); + msg.send(tcpFrontend); + } + } + + internalWorkerThread.interrupt(); + internalPublishDealerWorkerThread.interrupt(); + if (!internalWorkerThread.isInterrupted()) { + internalWorkerThread.join(); + } + if (!internalPublishDealerWorkerThread.isInterrupted()) { + internalPublishDealerWorkerThread.join(); + } + } catch (InterruptedException | IllegalStateException e) { + // terminated broker via interrupt + } + } + + private static class InternalWorker implements Runnable { + private final ZContext ctx; + public InternalWorker(ZContext ctx) { + this.ctx = ctx; + } + + @Override + public void run() { + try (Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(WORKER_ID); + worker.connect("inproc://broker"); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + msg.send(worker); + } + } catch (ZMQException e) { + // terminate internal worker + } + } + } + + private static class InternalPublishDealerWorker implements Runnable { + private final ZContext ctx; + public InternalPublishDealerWorker(ZContext ctx) { + this.ctx = ctx; + } + + @Override + public void run() { + try (Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(PUBLISH_ID); + worker.connect("inproc://broker"); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + if (START.equals(msg.getFirst().getString(ZMQ.CHARSET))) { + // System.err.println("dealer (indirect): start pushing"); + for (int requests = 0; requests < sampleSizePub; requests++) { + worker.send(SUB_TOPIC, ZMQ.SNDMORE); + worker.send(SUB_DATA); + } + } + } + } catch (ZMQException | IllegalStateException e) { + // terminate internal publish worker + } + } + } + } + + protected static class Worker implements Runnable { + @Override + public void run() { + try (ZContext ctx = new ZContext(); Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(WORKER_ID); + worker.connect(ENDPOINT_TCP); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + msg.send(worker); + } + } catch (ZMQException e) { + // terminate worker + } + } + } + + private static class PublishWorker implements Runnable { + @Override + public void run() { + try (ZContext ctx = new ZContext(); Socket worker = ctx.createSocket(SocketType.PUB)) { + worker.setHWM(0); + worker.bind(ENDPOINT_PUBSUB); + // System.err.println("PublishWorker: start publishing"); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + worker.send(SUB_TOPIC, ZMQ.SNDMORE); + worker.send(SUB_DATA); + } + } catch (ZMQException | IllegalStateException e) { + // terminate pub-Dealer worker + } + } + } + + private static class PublishDealerWorker implements Runnable { + @Override + public void run() { + try (ZContext ctx = new ZContext(); Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(PUBLISH_ID); + //worker.bind("tcp://localhost:5558"); + worker.connect("tcp://localhost:5556"); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + if (START.equals(msg.getFirst().getString(ZMQ.CHARSET))) { + // System.err.println("dealer (indirect): start pushing"); + for (int requests = 0; requests < sampleSizePub; requests++) { + worker.send(SUB_TOPIC, ZMQ.SNDMORE); + worker.send(SUB_DATA); + } + } + } + } catch (ZMQException | IllegalStateException e) { + // terminate publish worker + } + } + } + + private static class PublishDirectDealerWorker implements Runnable { + @Override + public void run() { + try (ZContext ctx = new ZContext(); Socket worker = ctx.createSocket(SocketType.DEALER)) { + worker.setHWM(0); + worker.setIdentity(PUBLISH_ID); + worker.bind("tcp://localhost:5558"); + while (RUN.get() && !Thread.currentThread().isInterrupted()) { + ZMsg msg = ZMsg.recvMsg(worker); + if (START.equals(msg.getFirst().getString(ZMQ.CHARSET))) { + // System.err.println("dealer (direct): start pushing"); + for (int requests = 0; requests < sampleSizePub; requests++) { + worker.send(SUB_TOPIC, ZMQ.SNDMORE); + worker.send(SUB_DATA); + } + } + } + } catch (ZMQException | IllegalStateException e) { + // terminate publish worker + } + } + } + + private static class Client implements Runnable { + @Override + public void run() { + try (ZContext ctx = new ZContext(); + Socket client = ctx.createSocket(SocketType.DEALER); + Socket subClient = ctx.createSocket(SocketType.SUB)) { + client.setHWM(0); + client.setIdentity(CLIENT_ID); + client.connect(ENDPOINT_ROUTER); + subClient.setHWM(0); + subClient.connect(ENDPOINT_PUBSUB); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); + + LOGGER.atDebug().log("Setting up test"); + // check and wait until broker is up and running + ZMsg.newStringMsg(TAG_EXTERNAL_STRING).addString(DUMMY_DATA).send(client); + ZMsg.recvMsg(client).destroy(); + + measure("Synchronous round-trip test (TCP)", sampleSize, () -> { + ZMsg req = new ZMsg(); + req.addString(TAG_EXTERNAL_STRING); + req.addString(DUMMY_DATA); + req.send(client); + ZMsg.recvMsg(client).destroy(); + }); + + measure("Synchronous round-trip test (InProc)", sampleSize, () -> { + ZMsg req = new ZMsg(); + req.addString(TAG_EXTERNAL_INTERNAL); + req.addString(DUMMY_DATA); + req.send(client); + ZMsg.recvMsg(client).destroy(); + }); + + measure("Asynchronous round-trip test (TCP)", sampleSize, () -> { + // send messages + ZMsg req = new ZMsg(); + req.addString(TAG_EXTERNAL_STRING); + req.addString(DUMMY_DATA); + req.send(client); }, () -> { + // receive messages + ZMsg.recvMsg(client).destroy(); }); + + measure("Asynchronous round-trip test (InProc)", sampleSize, () -> { + // send messages + ZMsg req = new ZMsg(); + req.addString(TAG_EXTERNAL_INTERNAL); + req.addString(DUMMY_DATA); + req.send(client); }, () -> { + // receive messages + ZMsg.recvMsg(client).destroy(); }); + + subClient.subscribe(SUB_TOPIC.getBytes(ZMQ.CHARSET)); + // first loop to empty potential queues/HWM + for (int requests = 0; requests < sampleSizePub; requests++) { + ZMsg req = ZMsg.recvMsg(subClient); + req.destroy(); + } + // start actual subscription loop + measure("Subscription (SUB) test", sampleSizePub, () -> { + ZMsg req = ZMsg.recvMsg(subClient); + req.destroy(); + }); + subClient.unsubscribe(SUB_TOPIC.getBytes(ZMQ.CHARSET)); + + client.disconnect(ENDPOINT_ROUTER); + client.setIdentity(SUBSCRIBER_ID); + client.connect(ENDPOINT_ROUTER); + ZMsg.newStringMsg(START).addFirst(TAG_EXTERNAL_STRING).send(client); + measure("Subscription (DEALER) test (TCP)", sampleSizePub, () -> { + ZMsg req = ZMsg.recvMsg(client); + req.destroy(); + }); + + ZMsg.newStringMsg(START).addFirst(TAG_EXTERNAL_INTERNAL).send(client); + measure("Subscription (DEALER) test (InProc)", sampleSizePub, () -> { + ZMsg req = ZMsg.recvMsg(client); + req.destroy(); + }); + + client.disconnect(ENDPOINT_ROUTER); + client.connect("tcp://localhost:5558"); + ZMsg.newStringMsg(START).send(client); + measure("Subscription (direct DEALER) test", sampleSizePub, () -> { + ZMsg req = ZMsg.recvMsg(client); + req.destroy(); + }); + + } catch (ZMQException e) { + LOGGER.atError().setCause(e).log("terminate client"); + } + } + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/aggregate/DemuxEventDispatcherTest.java b/concepts/src/test/java/io/opencmw/concepts/aggregate/DemuxEventDispatcherTest.java new file mode 100644 index 00000000..4fcc2bcf --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/aggregate/DemuxEventDispatcherTest.java @@ -0,0 +1,97 @@ +package io.opencmw.concepts.aggregate; + +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.awaitility.Awaitility; +import org.hamcrest.Matcher; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.TimeoutBlockingWaitStrategy; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.EventHandlerGroup; +import com.lmax.disruptor.dsl.ProducerType; +import com.lmax.disruptor.util.DaemonThreadFactory; + +/** + * + * @author Alexander Krimm + */ +@SuppressWarnings("unchecked") +class DemuxEventDispatcherTest { + static Stream workingEventSamplesProvider() { + return Stream.of( + arguments("ordinary", "a1 b1 c1 a2 b2 c2 a3 b3 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("duplicate events", "a1 b1 c1 b1 a2 b2 c2 a2 a3 b3 c3 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("reordered", "a1 c1 b1 a2 b2 c2 a3 b3 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("interleaved", "a1 b1 a2 b2 c1 a3 b3 c2 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("missing event", "a1 b1 a2 b2 c2 a3 b3 c3", "a2 b2 c2; a3 b3 c3", "1", 1), + arguments("missing device", "a1 b1 a2 b2 a3 b3", "", "1 2 3", 1), + arguments("late", "a1 b1 a2 b2 c2 a3 b3 c3 c1", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("timeout without event", "a1 b1 c1 a2 b2", "a1 b1 c1", "2", 1), + arguments("long queue", "a1 b1 c1 a2 b2", "a1 b1 c1; a1001 b1001 c1001; a2001 b2001 c2001; a3001 b3001 c3001; a4001 b4001 c4001", "2 1002 2002 3002 4002", 5), + arguments("simple broken long queue", "a1 b1", "", "1 1001 2001 3001 4001", 5), + arguments("single event timeout", "a1 b1 pause pause c1", "", "1", 1)); + } + + @ParameterizedTest + @MethodSource("workingEventSamplesProvider") + void testSimpleEvents(final String eventSetupName, final String events, final String aggregatesAll, final String timeoutsAll, final int repeat) { + // handler which collects all aggregate events which are republished to the buffer + final Set> aggResults = ConcurrentHashMap.newKeySet(); + final Set aggTimeouts = ConcurrentHashMap.newKeySet(); + EventHandler testHandler = (ev, seq, eob) -> { + System.out.println(ev); + if (ev.payload instanceof List) { + @SuppressWarnings("unchecked") + final List agg = (List) ev.payload; + final Set payloads = agg.stream().map(e -> (String) ((TestEventSource.Event) e.payload).payload).collect(Collectors.toSet()); + aggResults.add(payloads); + } + if (ev.payload instanceof String && ((String) ev.payload).startsWith("aggregation timed out for bpcts: ")) { + final String payload = ((String) ev.payload); + aggTimeouts.add(Integer.parseInt(payload.substring(33, payload.indexOf(' ', 34)))); + } + }; + + // create event ring buffer and add de-multiplexing processors + final Disruptor disruptor = new Disruptor<>( + TestEventSource.IngestedEvent::new, + 256, + DaemonThreadFactory.INSTANCE, + ProducerType.MULTI, + new TimeoutBlockingWaitStrategy(200, TimeUnit.MILLISECONDS)); + final DemuxEventDispatcher aggProc = new DemuxEventDispatcher(disruptor.getRingBuffer()); + final EventHandlerGroup endBarrier = disruptor.handleEventsWith(testHandler).handleEventsWith(aggProc).then(aggProc.getAggregationHander()); + RingBuffer rb = disruptor.start(); + + // Use event source to publish demo events to the buffer. + TestEventSource testEventSource = new TestEventSource(events, repeat, rb); + Assertions.assertDoesNotThrow(testEventSource::run); + + // wait for all events to be played and processed + Awaitility.await().atMost(Duration.ofSeconds(repeat)).until(() -> endBarrier.asSequenceBarrier().getCursor() == rb.getCursor() && Arrays.stream(aggProc.getAggregationHander()).allMatch(w -> w.bpcts == -1)); + // compare aggregated results and timeouts + MatcherAssert.assertThat(aggResults, Matchers.containsInAnyOrder(Arrays.stream(aggregatesAll.split(";")) + .filter(s -> !s.isEmpty()) + .map(s -> Matchers.containsInAnyOrder(Arrays.stream(s.split(" ")).map(String::trim).filter(e -> !e.isEmpty()).toArray())) + .toArray(Matcher[] ::new))); + System.out.println(aggTimeouts); + MatcherAssert.assertThat(aggTimeouts, Matchers.containsInAnyOrder(Arrays.stream(timeoutsAll.split(" ")).filter(s -> !s.isEmpty()).map(Integer::parseInt).toArray(Integer[] ::new))); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/majordomo/ClientSampleV1.java b/concepts/src/test/java/io/opencmw/concepts/majordomo/ClientSampleV1.java new file mode 100644 index 00000000..f097a5f2 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/majordomo/ClientSampleV1.java @@ -0,0 +1,48 @@ +package io.opencmw.concepts.majordomo; + +import java.nio.charset.StandardCharsets; + +import org.zeromq.ZMsg; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacToken; + +/** +* Majordomo Protocol client example. Uses the mdcli API to hide all OpenCmwProtocol aspects +*/ +public final class ClientSampleV1 { // nomen est omen + private static final int N_SAMPLES = 50_000; + + private ClientSampleV1() { + // requires only static methods for testing + } + + public static void main(String[] args) { + MajordomoClientV1 clientSession = new MajordomoClientV1("tcp://localhost:5555", "customClientName"); + final byte[] serviceBytes = "mmi.echo".getBytes(StandardCharsets.UTF_8); + // final byte[] serviceBytes = "inproc.echo".getBytes(StandardCharsets.UTF_8) + // final byte[] serviceBytes = "echo".getBytes(StandardCharsets.UTF_8) + + int count; + long start = System.currentTimeMillis(); + for (count = 0; count < N_SAMPLES; count++) { + final String requestMsg = "Hello world - sync - " + count; + final byte[] request = requestMsg.getBytes(StandardCharsets.UTF_8); + final byte[] rbacToken = new RbacToken(BasicRbacRole.ADMIN, "HASHCODE").getBytes(); // NOPMD + final ZMsg reply = clientSession.send(serviceBytes, request, rbacToken); // with RBAC + // final ZMsg reply = clientSession.send(serviceBytes, request); // w/o RBAC + if (count < 10 || count % 10_000 == 0 || count >= (N_SAMPLES - 10)) { + System.err.println("client iteration = " + count + " - received: " + reply); + } + if (reply == null) { + break; // Interrupt or failure + } + reply.destroy(); + } + + long mark1 = System.currentTimeMillis(); + double diff2 = 1e-3 * (mark1 - start); + System.err.printf("%d requests/replies processed in %d ms -> %f op/s\n", count, mark1 - start, count / diff2); + clientSession.destroy(); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/majordomo/ClientSampleV2.java b/concepts/src/test/java/io/opencmw/concepts/majordomo/ClientSampleV2.java new file mode 100644 index 00000000..b08c98b4 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/majordomo/ClientSampleV2.java @@ -0,0 +1,50 @@ +package io.opencmw.concepts.majordomo; + +import java.nio.charset.StandardCharsets; + +import org.zeromq.ZMsg; + +/** + * Majordomo Protocol client example, asynchronous. Uses the mdcli API to hide + * all OpenCmwProtocol aspects + */ + +public final class ClientSampleV2 { // NOPMD -- nomen est omen + private static final int N_SAMPLES = 1_000_000; + + private ClientSampleV2() { + // requires only static methods for testing + } + + public static void main(String[] args) { + MajordomoClientV2 clientSession = new MajordomoClientV2("tcp://localhost:5555"); + final byte[] serviceBytes = "mmi.echo".getBytes(StandardCharsets.UTF_8); + // final byte[] serviceBytes = "inproc.echo".getBytes(StandardCharsets.UTF_8) + // final byte[] serviceBytes = "echo".getBytes(StandardCharsets.UTF_8) + + int count; + long start = System.currentTimeMillis(); + for (count = 0; count < N_SAMPLES; count++) { + final String requestMsg = "Hello world - async - " + count; + clientSession.send(serviceBytes, requestMsg.getBytes(StandardCharsets.UTF_8)); + } + long mark1 = System.currentTimeMillis(); + double diff1 = 1e-3 * (mark1 - start); + System.err.printf("%d requests processed in %d ms -> %f op/s\n", count, mark1 - start, count / diff1); + + for (count = 0; count < N_SAMPLES; count++) { + ZMsg reply = clientSession.recv(); + if (count < 10 || count % 100_000 == 0 || count >= (N_SAMPLES - 10)) { + System.err.println("client iteration = " + count + " - received: " + reply); + } + if (reply == null) { + break; // Interrupt or failure + } + reply.destroy(); + } + long mark2 = System.currentTimeMillis(); + double diff2 = 1e-3 * (mark2 - start); + System.err.printf("%d requests/replies processed in %d ms -> %f op/s\n", count, mark2 - start, count / diff2); + clientSession.destroy(); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/majordomo/MajordomoBrokerTests.java b/concepts/src/test/java/io/opencmw/concepts/majordomo/MajordomoBrokerTests.java new file mode 100644 index 00000000..bee46941 --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/majordomo/MajordomoBrokerTests.java @@ -0,0 +1,253 @@ +package io.opencmw.concepts.majordomo; + +import static org.junit.jupiter.api.Assertions.*; + +import static io.opencmw.concepts.majordomo.MajordomoProtocol.*; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.awaitility.Awaitility; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.zeromq.SocketType; +import org.zeromq.Utils; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacToken; + +public class MajordomoBrokerTests { + private static final byte[] DEFAULT_RBAC_TOKEN = new RbacToken(BasicRbacRole.ADMIN, "HASHCODE").getBytes(); + private static final byte[] DEFAULT_MMI_SERVICE = "mmi.service".getBytes(StandardCharsets.UTF_8); + private static final byte[] DEFAULT_ECHO_SERVICE = "mmi.echo".getBytes(StandardCharsets.UTF_8); + private static final String DEFAULT_REQUEST_MESSAGE = "Hello World!"; + private static final byte[] DEFAULT_REQUEST_MESSAGE_BYTES = DEFAULT_REQUEST_MESSAGE.getBytes(StandardCharsets.UTF_8); + + @Test + public void basicLowLevelRequestReplyTest() throws InterruptedException, IOException { + MajordomoBroker broker = new MajordomoBroker(1, BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final int openPort = Utils.findOpenPort(); + broker.bind("tcp://*:" + openPort); + assertFalse(broker.isRunning(), "broker not running"); + broker.start(); + assertTrue(broker.isRunning(), "broker running"); + // test interfaces + assertNotNull(broker.getContext()); + assertNotNull(broker.getInternalRouterSocket()); + assertNotNull(broker.getServices()); + assertEquals(2, broker.getServices().size()); + assertDoesNotThrow(() -> broker.addInternalService(new MajordomoWorker(broker.getContext(), "demoService"), 10)); + assertEquals(3, broker.getServices().size()); + assertDoesNotThrow(() -> broker.removeService("demoService")); + assertEquals(2, broker.getServices().size()); + + // wait until all services are initialised + Thread.sleep(200); + + final ZMQ.Socket clientSocket = broker.getContext().createSocket(SocketType.DEALER); + clientSocket.setIdentity("demoClient".getBytes(StandardCharsets.UTF_8)); + clientSocket.connect("tcp://localhost:" + openPort); + + // wait until client is connected + Thread.sleep(200); + + sendClientMessage(clientSocket, MdpClientCommand.C_UNKNOWN, null, DEFAULT_ECHO_SERVICE, DEFAULT_REQUEST_MESSAGE_BYTES); + + final MdpMessage reply = receiveMdpMessage(clientSocket); + assertNotNull(reply.toString()); + assertNotNull(reply, "reply message w/o RBAC token not being null"); + assertTrue(reply instanceof MdpClientMessage); + MdpClientMessage clientMessage = (MdpClientMessage) reply; + assertNull(clientMessage.senderID); // default dealer socket does not export sender ID (only ROUTER and/or enabled sockets) + assertEquals(MdpSubProtocol.C_CLIENT, clientMessage.protocol, "equal protocol"); + assertEquals(MdpClientCommand.C_UNKNOWN, clientMessage.command, "matching command"); + assertArrayEquals(DEFAULT_ECHO_SERVICE, clientMessage.serviceNameBytes, "equal service name"); + assertNotNull(clientMessage.payload, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, clientMessage.payload[0], "equal data"); + assertFalse(clientMessage.hasRbackToken()); + assertNotNull(clientMessage.getRbacFrame()); + assertArrayEquals(new byte[0], clientMessage.getRbacFrame()); + + broker.stopBroker(); + } + + @Test + public void basicSynchronousRequestReplyTest() throws InterruptedException, IOException { + MajordomoBroker broker = new MajordomoBroker(1, BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final int openPort = Utils.findOpenPort(); + broker.bind("tcp://*:" + openPort); + broker.start(); + assertEquals(2, broker.getServices().size()); + + // add external (albeit inproc) Majordomo worker to the broker + MajordomoWorker internal = new MajordomoWorker(broker.getContext(), "inproc.echo", BasicRbacRole.ADMIN); + internal.registerHandler(input -> input); // output = input : echo service is complex :-) + internal.start(); + + // add external Majordomo worker to the broker + MajordomoWorker external = new MajordomoWorker(broker.getContext(), "ext.echo", BasicRbacRole.ADMIN); + external.registerHandler(input -> input); // output = input : echo service is complex :-) + external.start(); + + // add external (albeit inproc) Majordomo worker to the broker + MajordomoWorker exceptionService = new MajordomoWorker(broker.getContext(), "inproc.exception", BasicRbacRole.ADMIN); + exceptionService.registerHandler(input -> { throw new IllegalAccessError("autsch"); }); // allways throw an exception + exceptionService.start(); + + // wait until all services are initialised + Thread.sleep(200); + assertEquals(5, broker.getServices().size()); + + // using simple synchronous client + MajordomoClientV1 clientSession = new MajordomoClientV1("tcp://localhost:" + openPort, "customClientName"); + assertEquals(3, clientSession.getRetries()); + assertDoesNotThrow(() -> clientSession.setRetries(4)); + assertEquals(4, clientSession.getRetries()); + assertEquals(2500, clientSession.getTimeout()); + assertDoesNotThrow(() -> clientSession.setTimeout(2000)); + assertEquals(2000, clientSession.getTimeout()); + assertNotNull(clientSession.getUniqueID()); + + { + final byte[] serviceBytes = "mmi.echo".getBytes(StandardCharsets.UTF_8); + final ZMsg replyWithoutRbac = clientSession.send(serviceBytes, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.pollLast().getData(), "equal data"); + } + + { + final byte[] serviceBytes = "inproc.echo".getBytes(StandardCharsets.UTF_8); + final ZMsg replyWithoutRbac = clientSession.send(serviceBytes, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.pollLast().getData(), "equal data"); + } + + { + final byte[] serviceBytes = "ext.echo".getBytes(StandardCharsets.UTF_8); + final ZMsg replyWithoutRbac = clientSession.send(serviceBytes, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.pollLast().getData(), "equal data"); + } + + { + final byte[] serviceBytes = "inproc.exception".getBytes(StandardCharsets.UTF_8); + final ZMsg replyWithoutRbac = clientSession.send(serviceBytes, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + // future: check exception type + } + + { + final byte[] serviceBytes = "mmi.echo".getBytes(StandardCharsets.UTF_8); + final ZMsg replyWithRbac = clientSession.send(serviceBytes, DEFAULT_REQUEST_MESSAGE_BYTES, DEFAULT_RBAC_TOKEN); // with RBAC + assertNotNull(replyWithRbac, "reply message with RBAC token not being null"); + assertNotNull(replyWithRbac.peekLast(), "RBAC token not being null"); + assertArrayEquals(DEFAULT_RBAC_TOKEN, replyWithRbac.pollLast().getData(), "equal RBAC token"); + assertNotNull(replyWithRbac.peekLast(), "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithRbac.pollLast().getData(), "equal data"); + } + + internal.stopWorker(); + external.stopWorker(); + exceptionService.stopWorker(); + broker.stopBroker(); + } + + @Test + public void basicMmiTests() throws IOException { + MajordomoBroker broker = new MajordomoBroker(1, BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final int openPort = Utils.findOpenPort(); + broker.bind("tcp://*:" + openPort); + broker.start(); + + // using simple synchronous client + MajordomoClientV1 clientSession = new MajordomoClientV1("tcp://localhost:" + openPort, "customClientName"); + + { + final ZMsg replyWithoutRbac = clientSession.send("mmi.echo", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.pollLast().getData(), "MMI echo service request"); + } + + { + final ZMsg replyWithoutRbac = clientSession.send(DEFAULT_MMI_SERVICE, DEFAULT_MMI_SERVICE); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertEquals("200", new String(replyWithoutRbac.pollLast().getData(), StandardCharsets.UTF_8), "known MMI service request"); + } + + { + final ZMsg replyWithoutRbac = clientSession.send(DEFAULT_MMI_SERVICE, DEFAULT_ECHO_SERVICE); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertEquals("200", new String(replyWithoutRbac.pollLast().getData(), StandardCharsets.UTF_8), "known MMI service request"); + } + + { + // MMI service request: service should not exist + final ZMsg replyWithoutRbac = clientSession.send(DEFAULT_MMI_SERVICE, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertEquals("400", new String(replyWithoutRbac.pollLast().getData(), StandardCharsets.UTF_8), "unknown MMI service request"); + } + + { + // unknown service name + final ZMsg replyWithoutRbac = clientSession.send("unknownService".getBytes(StandardCharsets.UTF_8), DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.peekLast(), "user-data not being null"); + assertEquals("501", new String(replyWithoutRbac.pollLast().getData()), "unknown service"); + } + + broker.stopBroker(); + } + + @Test + public void basicASynchronousRequestReplyTest() throws IOException { + MajordomoBroker broker = new MajordomoBroker(1, BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final int openPort = Utils.findOpenPort(); + broker.bind("tcp://*:" + openPort); + broker.start(); + + final AtomicInteger counter = new AtomicInteger(0); + new Thread(() -> { + // using simple synchronous client + MajordomoClientV2 clientSession = new MajordomoClientV2("tcp://localhost:" + openPort); + assertEquals(2500, clientSession.getTimeout()); + assertDoesNotThrow(() -> clientSession.setTimeout(2000)); + assertEquals(2000, clientSession.getTimeout()); + + // send bursts of 10 messages + for (int i = 0; i < 5; i++) { + clientSession.send("mmi.echo", DEFAULT_REQUEST_MESSAGE_BYTES); + clientSession.send(DEFAULT_ECHO_SERVICE, DEFAULT_REQUEST_MESSAGE_BYTES); + } + + // send bursts of 10 messages + for (int i = 0; i < 10; i++) { + final ZMsg reply = clientSession.recv(); + assertNotNull(reply, "reply message w/o RBAC token not being null"); + assertNotNull(reply.peekLast(), "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, reply.getLast().getData()); + counter.getAndIncrement(); + } + }).start(); + + Awaitility.await().alias("wait for reply messages").atMost(1, TimeUnit.SECONDS).until(counter::get, Matchers.equalTo(10)); + assertEquals(10, counter.get(), "received expected number of replies"); + + broker.stopBroker(); + } +} diff --git a/concepts/src/test/java/io/opencmw/concepts/majordomo/SimpleEchoServiceWorker.java b/concepts/src/test/java/io/opencmw/concepts/majordomo/SimpleEchoServiceWorker.java new file mode 100644 index 00000000..3e82007f --- /dev/null +++ b/concepts/src/test/java/io/opencmw/concepts/majordomo/SimpleEchoServiceWorker.java @@ -0,0 +1,21 @@ +package io.opencmw.concepts.majordomo; + +import io.opencmw.rbac.BasicRbacRole; + +/** +* Majordomo Protocol worker example. Uses the mdwrk API to hide all OpenCmwProtocol aspects +* +*/ +public class SimpleEchoServiceWorker { // NOPMD - nomen est omen + + private SimpleEchoServiceWorker() { + // private helper/test class + } + + public static void main(String[] args) { + MajordomoWorker workerSession = new MajordomoWorker("tcp://localhost:5556", "echo", BasicRbacRole.ADMIN); + // workerSession.setDaemon(true); // use this if running in another app that controls threads + workerSession.registerHandler(input -> input); // output = input : echo service is complex :-) + workerSession.start(); + } +} diff --git a/concepts/src/test/resources/simplelogger.properties b/concepts/src/test/resources/simplelogger.properties new file mode 100644 index 00000000..a01ef764 --- /dev/null +++ b/concepts/src/test/resources/simplelogger.properties @@ -0,0 +1,50 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=debug + +# The output target which can be the path to a file, or the special values "System.out" and "System.err". +# Default is "System.err". +org.slf4j.simpleLogger.logFile=System.out + +# If the output target is set to "System.out" or "System.err" (see preceding entry), by default, +# logs will be output to the latest value referenced by System.out/err variables. +# By setting this parameter to true, the output stream will be cached, i.e. assigned once at initialization +# time and re-used independently of the current value referenced by System.out/err. +org.slf4j.simpleLogger.cacheOutputStream=true + +# Logging detail level for a SimpleLogger instance named "a.b.c". Right-side value must be one of +# "trace", "debug", "info", "warn", "error" or "off". When a SimpleLogger named "a.b.c" is initialized, +# its level is assigned from this property. If unspecified, the level of nearest parent logger will be used, +# and if none is set, then the value specified by org.slf4j.simpleLogger.defaultLogLevel will be used. +org.slf4j.simpleLogger.log.de.gsi.* + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +# org.slf4j.simpleLogger.showThreadName=false + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +# org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/config/hooks/pre-commit b/config/hooks/pre-commit new file mode 100755 index 00000000..4201c858 --- /dev/null +++ b/config/hooks/pre-commit @@ -0,0 +1,59 @@ +#!/bin/bash +#enforces .clang-format style guide prior to committing to the git repository + +CLANG_MIN_VERSION="9.0.0" + +set -e + +REPO_ROOT_DIR="$(git rev-parse --show-toplevel)" +CLANG_FORMAT="$(command -v clang-format)" +CLANG_VERSION="$(${CLANG_FORMAT} --version | sed '/^clang-format version /!d;s///;s/-.*//;s///g')" + +compare_version () { + echo " " + if [[ $1 == $2 ]] + then + CLANG_MIN_VERSION_MATCH="=" + return + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + CLANG_MIN_VERSION_MATCH="<" + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + CLANG_MIN_VERSION_MATCH=">" + return + fi + done + CLANG_MIN_VERSION_MATCH="=" + return +} + +compare_version ${CLANG_MIN_VERSION} ${CLANG_VERSION} + +files=$((git diff --cached --name-only --diff-filter=ACMR | grep -Ei "\.(c|cc|cpp|cxx|c\+\+|h|hh|hpp|hxx|h\+\+|java)$") || true) +if [ -n "${files}" ]; then + + if [ -n "${CLANG_FORMAT}" ] && [ "$CLANG_MIN_VERSION_MATCH" != "<" ]; then + spaced_files=$(echo "$files" | paste -s -d " " -) + # echo "reformatting ${spaced_files}" + "${CLANG_FORMAT}" -style=file -i $spaced_files >/dev/null + fi + git add $(echo "$files" | paste -s -d " " -) +fi \ No newline at end of file diff --git a/config/hooks/pre-commit-stub b/config/hooks/pre-commit-stub new file mode 100755 index 00000000..d168dc2f --- /dev/null +++ b/config/hooks/pre-commit-stub @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# stub pre-commit hook +# just a runner for the real pre-commit script +# if script cannot be found, exit without error +# (to not block local commits) + +set -e + +REPO_ROOT_DIR="$(git rev-parse --show-toplevel)" +PRE_COMMIT_SCRIPT="${REPO_ROOT_DIR}/config/hooks/pre-commit" + +if [ -f $PRE_COMMIT_SCRIPT ]; then + source $PRE_COMMIT_SCRIPT +fi \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml new file mode 100644 index 00000000..aaa10e58 --- /dev/null +++ b/core/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + + + io.opencmw + opencmw + ${revision}${sha1}${changelist} + ../pom.xml + + + core + + + ZeroMQ and REST-based micro-service implementation. + + + + + io.opencmw + serialiser + ${revision}${sha1}${changelist} + + + de.gsi.dataset + chartfx-dataset + ${version.chartfx} + true + test + + + org.zeromq + jeromq + ${version.jeromq} + + + com.lmax + disruptor + ${versions.lmax.disruptor} + + + + org.apache.commons + commons-lang3 + ${version.commons-lang3} + + + + + diff --git a/core/src/main/java/io/opencmw/AggregateEventHandler.java b/core/src/main/java/io/opencmw/AggregateEventHandler.java new file mode 100644 index 00000000..29acdd75 --- /dev/null +++ b/core/src/main/java/io/opencmw/AggregateEventHandler.java @@ -0,0 +1,359 @@ +package io.opencmw; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.utils.Cache; +import io.opencmw.utils.NoDuplicatesList; +import io.opencmw.utils.SharedPointer; + +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.Sequence; +import com.lmax.disruptor.SequenceReportingEventHandler; +import com.lmax.disruptor.TimeoutHandler; + +/** + * Dispatches aggregation workers upon seeing new values for a specified event field. + * Each aggregation worker then assembles all events for this value and optionally publishes back an aggregated events. + * If the aggregation is not completed within a configurable timeout, a partial AggregationEvent is published. + * + * For now events are aggregated into a list of Objects until a certain number of events is reached. + * The final api should allow to specify different Objects to be placed into a result domain object. + * + * @author Alexander Krimm + */ +@SuppressWarnings("PMD.LinguisticNaming") // fluent-style API with setter returning factory +public class AggregateEventHandler implements SequenceReportingEventHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(AggregateEventHandler.class); + // private Map aggregatedBpcts = new SoftHashMap<>(RETENTION_SIZE); + protected final Map aggregatedBpcts; + private final RingBuffer ringBuffer; + private final long timeOut; + private final TimeUnit timeOutUnit; + private final int numberOfEventsToAggregate; + private final List deviceList; + private final List> evtTypeFilter; + private final InternalAggregationHandler[] aggregationHandler; + private final List freeWorkers; + private final String aggregateName; + private Sequence seq; + + private AggregateEventHandler(final RingBuffer ringBuffer, final String aggregateName, final long timeOut, final TimeUnit timeOutUnit, final int nWorkers, final int retentionSize, final List deviceList, List> evtTypeFilter) { // NOPMD NOSONAR -- number of arguments acceptable/ complexity handled by factory + this.ringBuffer = ringBuffer; + this.aggregateName = aggregateName; + this.timeOut = timeOut; + this.timeOutUnit = timeOutUnit; + + freeWorkers = Collections.synchronizedList(new ArrayList<>(nWorkers)); + aggregationHandler = new InternalAggregationHandler[nWorkers]; + for (int i = 0; i < nWorkers; i++) { + aggregationHandler[i] = new InternalAggregationHandler(); + freeWorkers.add(aggregationHandler[i]); + } + aggregatedBpcts = new Cache<>(retentionSize); + this.deviceList = deviceList; + this.evtTypeFilter = evtTypeFilter; + numberOfEventsToAggregate = deviceList.size() + evtTypeFilter.size(); + } + + public InternalAggregationHandler[] getAggregationHander() { + return aggregationHandler; + } + + @Override + public void onEvent(final RingBufferEvent event, final long nextSequence, final boolean b) { + final TimingCtx ctx = event.getFilter(TimingCtx.class); + if (ctx == null) { + return; + } + + // final boolean alreadyScheduled = Arrays.stream(workers).filter(w -> w.bpcts == eventBpcts).findFirst().isPresent(); + final boolean alreadyScheduled = aggregatedBpcts.containsKey(ctx.bpcts); + if (alreadyScheduled) { + return; + } + while (true) { + if (!freeWorkers.isEmpty()) { + final InternalAggregationHandler freeWorker = freeWorkers.remove(0); + freeWorker.bpcts = ctx.bpcts; + freeWorker.aggStart = event.arrivalTimeStamp; + aggregatedBpcts.put(ctx.bpcts, new Object()); // NOPMD - necessary to allocate inside loop + seq.set(nextSequence); // advance sequence to let workers process events up to here + return; + } + // no free worker available + long waitTimeNanos = Long.MAX_VALUE; + for (InternalAggregationHandler w : aggregationHandler) { + final long currentTime = System.currentTimeMillis(); + final long diffMillis = currentTime - w.aggStart; + waitTimeNanos = Math.min(waitTimeNanos, TimeUnit.MILLISECONDS.toNanos(diffMillis)); + if (w.bpcts != -1 && diffMillis < timeOutUnit.toMillis(timeOut)) { + w.publishAndFreeWorker(EvtTypeFilter.UpdateType.PARTIAL); // timeout reached, publish partial result and free worker + break; + } + } + LockSupport.parkNanos(waitTimeNanos); + } + } + + @Override + public void setSequenceCallback(final Sequence sequence) { + this.seq = sequence; + } + + public static AggregateEventHandlerFactory getFactory() { + return new AggregateEventHandlerFactory(); + } + + public static class AggregateEventHandlerFactory { + private final List deviceList = new NoDuplicatesList<>(); + private final List> evtTypeFilter = new NoDuplicatesList<>(); + private RingBuffer ringBuffer; + private int numberWorkers = 4; // number of workers defines the maximum number of aggregate events groups which can be overlapping + private long timeOut = 400; + private TimeUnit timeOutUnit = TimeUnit.MILLISECONDS; + private int retentionSize = 12; + private String aggregateName; + + public AggregateEventHandler build() { + if (aggregateName == null || aggregateName.isBlank()) { + throw new IllegalArgumentException("aggregateName must not be null or blank"); + } + if (ringBuffer == null) { + throw new IllegalArgumentException("ringBuffer must not be null"); + } + final int actualRetentionSize = Math.min(retentionSize, 3 * numberWorkers); + return new AggregateEventHandler(ringBuffer, aggregateName, timeOut, timeOutUnit, numberWorkers, actualRetentionSize, deviceList, evtTypeFilter); + } + + public String getAggregateName() { + return aggregateName; + } + + public AggregateEventHandlerFactory setAggregateName(final String aggregateName) { + this.aggregateName = aggregateName; + return this; + } + + public List getDeviceList() { + return deviceList; + } + + /** + * + * @param deviceList lists of devices, event names, etc. that shall be aggregated + * @return itself (fluent design) + */ + public AggregateEventHandlerFactory setDeviceList(final List deviceList) { + this.deviceList.addAll(deviceList); + return this; + } + + /** + * + * @param devices single or lists of devices, event names, etc. that shall be aggregated + * @return itself (fluent design) + */ + public AggregateEventHandlerFactory setDeviceList(final String... devices) { + this.deviceList.addAll(Arrays.asList(devices)); + return this; + } + + public List> getEvtTypeFilter() { + return evtTypeFilter; + } + + /** + * + * @param evtTypeFilter single or lists of predicate filters of events that shall be aggregated + * @return itself (fluent design) + */ + @SafeVarargs + public final AggregateEventHandlerFactory setEvtTypeFilter(final Predicate... evtTypeFilter) { + this.evtTypeFilter.addAll(Arrays.asList(evtTypeFilter)); + return this; + } + + /** + * + * @param evtTypeFilter single or lists of predicate filters of events that shall be aggregated + * @return itself (fluent design) + */ + public AggregateEventHandlerFactory setEvtTypeFilter(final List> evtTypeFilter) { + this.evtTypeFilter.addAll(evtTypeFilter); + return this; + } + + /** + * @return number of workers defines the maximum number of aggregate events groups which can be overlapping + */ + public int getNumberWorkers() { + return numberWorkers; + } + + /** + * + * @param numberWorkers number of workers defines the maximum number of aggregate events groups which can be overlapping + * @return itself (fluent design) + */ + public AggregateEventHandlerFactory setNumberWorkers(final int numberWorkers) { + if (numberWorkers < 1) { + throw new IllegalArgumentException("numberWorkers must not be < 1: " + numberWorkers); + } + this.numberWorkers = numberWorkers; + return this; + } + + public int getRetentionSize() { + return retentionSize; + } + + public AggregateEventHandlerFactory setRetentionSize(final int retentionSize) { + if (retentionSize < 1) { + throw new IllegalArgumentException("timeOut must not be < 1: " + retentionSize); + } + this.retentionSize = retentionSize; + return this; + } + + public RingBuffer getRingBuffer() { + return ringBuffer; + } + + public AggregateEventHandlerFactory setRingBuffer(final RingBuffer ringBuffer) { + if (ringBuffer == null) { + throw new IllegalArgumentException("ringBuffer must not null"); + } + this.ringBuffer = ringBuffer; + return this; + } + + public long getTimeOut() { + return timeOut; + } + + public TimeUnit getTimeOutUnit() { + return timeOutUnit; + } + + public AggregateEventHandlerFactory setTimeOut(final long timeOut, final TimeUnit timeOutUnit) { + if (timeOut <= 0) { + throw new IllegalArgumentException("timeOut must not be <=0: " + timeOut); + } + if (timeOutUnit == null) { + throw new IllegalArgumentException("timeOutUnit must not null"); + } + this.timeOut = timeOut; + this.timeOutUnit = timeOutUnit; + return this; + } + } + + @SuppressWarnings("PMD.AvoidUsingVolatile") // cache-specific usage here + protected class InternalAggregationHandler implements EventHandler, TimeoutHandler { + protected volatile long bpcts = -1; // [ms] + protected volatile long aggStart = -1; // [ns] + protected List aggregatedEventsStash = new ArrayList<>(); + + @Override + public void onEvent(final RingBufferEvent event, final long sequence, final boolean endOfBatch) { + if (bpcts != -1 && event.arrivalTimeStamp > aggStart + timeOutUnit.toMillis(timeOut)) { + publishAndFreeWorker(EvtTypeFilter.UpdateType.PARTIAL); + return; + } + final TimingCtx ctx = event.getFilter(TimingCtx.class); + if (bpcts == -1 || ctx == null || ctx.bpcts != bpcts) { + return; // skip irrelevant events + } + final EvtTypeFilter evtType = event.getFilter(EvtTypeFilter.class); + if (evtType == null) { + throw new IllegalArgumentException("cannot aggregate events without ring buffer containing EvtTypeFilter"); + } + if ((!deviceList.isEmpty() && !deviceList.contains(evtType.typeName)) || (!evtTypeFilter.isEmpty() && evtTypeFilter.stream().noneMatch(filter -> filter.test(event)))) { + return; + } + + aggregatedEventsStash.add(event); + if (aggregatedEventsStash.size() == numberOfEventsToAggregate) { + publishAndFreeWorker(EvtTypeFilter.UpdateType.COMPLETE); + } + } + + @Override + public void onTimeout(final long sequence) { + if (bpcts != -1 && System.currentTimeMillis() > aggStart + timeOut) { + publishAndFreeWorker(EvtTypeFilter.UpdateType.PARTIAL); + } + } + + protected void publishAndFreeWorker(final EvtTypeFilter.UpdateType updateType) { + final long now = System.currentTimeMillis(); + ringBuffer.publishEvent(((event, sequence, arg0) -> { + final TimingCtx ctx = event.getFilter(TimingCtx.class); + if (ctx == null) { + throw new IllegalStateException("RingBufferEvent has not TimingCtx definition"); + } + final EvtTypeFilter evtType = event.getFilter(EvtTypeFilter.class); + if (evtType == null) { + throw new IllegalArgumentException("cannot aggregate events without ring buffer containing EvtTypeFilter"); + } + + // write header/meta-type data + event.arrivalTimeStamp = now; + event.payload = new SharedPointer<>(); + final Map> aggregatedItems = new HashMap<>(); // + event.payload.set(aggregatedItems); + if (updateType == EvtTypeFilter.UpdateType.PARTIAL) { + LOGGER.atInfo().log("aggregation timed out for bpcts: " + bpcts); + } + + if (aggregatedEventsStash.isEmpty()) { + // notify empty aggregate + event.parentSequenceNumber = sequence; + evtType.typeName = aggregateName; + evtType.updateType = EvtTypeFilter.UpdateType.EMPTY; + evtType.evtType = EvtTypeFilter.DataType.AGGREGATE_DATA; + return; + } + + // handle non-empty aggregate + final RingBufferEvent firstItem = aggregatedEventsStash.get(0); + for (int i = 0; i < firstItem.filters.length; i++) { + firstItem.filters[i].copyTo(event.filters[i]); + } + if (updateType == EvtTypeFilter.UpdateType.PARTIAL) { + LOGGER.atInfo().log("aggregation timed out for 2:bpcts: " + event.getFilter(TimingCtx.class).bpcts); + } + evtType.typeName = aggregateName; + evtType.updateType = updateType; + evtType.evtType = EvtTypeFilter.DataType.AGGREGATE_DATA; + event.parentSequenceNumber = sequence; + + // add new events to payload + for (RingBufferEvent rbEvent : aggregatedEventsStash) { + final EvtTypeFilter type = rbEvent.getFilter(EvtTypeFilter.class); + aggregatedItems.put(type.typeName, rbEvent.payload.getCopy()); + } + }), + aggregatedEventsStash); + + // init worker for next aggregation iteration + bpcts = -1; + aggregatedEventsStash = new ArrayList<>(); + freeWorkers.add(this); + } + } +} diff --git a/core/src/main/java/io/opencmw/EventStore.java b/core/src/main/java/io/opencmw/EventStore.java new file mode 100644 index 00000000..bf7b93bb --- /dev/null +++ b/core/src/main/java/io/opencmw/EventStore.java @@ -0,0 +1,450 @@ +package io.opencmw; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.utils.Cache; +import io.opencmw.utils.LimitedArrayList; +import io.opencmw.utils.NoDuplicatesList; +import io.opencmw.utils.WorkerThreadFactory; + +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.TimeoutBlockingWaitStrategy; +import com.lmax.disruptor.WaitStrategy; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.EventHandlerGroup; +import com.lmax.disruptor.dsl.ProducerType; +import com.lmax.disruptor.util.Util; + +/** + * Initial event-source concept with one primary event-stream and arbitrary number of secondary context-multiplexed event-streams. + * + * Each event-stream is implemented using LMAX's disruptor ring-buffer using the default {@link RingBufferEvent}. + * + * The multiplexing-context for the secondary ring buffers is controlled via the 'Function<RingBufferEvent, String> muxCtxFunction' + * function that produces a unique string hash for a given ring buffer event, e.g.: + * {@code Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid;} + * + * See EventStoreTest in the teest directory for usage and API examples. + * + * @author rstein + */ +public class EventStore { + private static final Logger LOGGER = LoggerFactory.getLogger(EventStore.class); + private static final String NOT_FOUND_FOR_MULTIPLEXING_CONTEXT = "disruptor not found for multiplexing context = "; + protected final WorkerThreadFactory threadFactory; + protected final List listener = new NoDuplicatesList<>(); + protected final List> allEventHandlers = new NoDuplicatesList<>(); + protected final List> muxCtxFunctions = new NoDuplicatesList<>(); + protected final Cache> eventStreams; + protected final Disruptor disruptor; + protected final int lengthHistoryBuffer; + protected Function> ctxMappingFunction; + + /** + * + * @param filterConfig static filter configuration + */ + @SafeVarargs + protected EventStore(final Cache.CacheBuilder> muxBuilder, final Function muxCtxFunction, final int ringBufferSize, final int lengthHistoryBuffer, final int maxThreadNumber, final boolean isSingleProducer, final WaitStrategy waitStrategy, final Class... filterConfig) { // NOPMD NOSONAR - handled by factory + assert filterConfig != null; + if (muxCtxFunction != null) { + this.muxCtxFunctions.add(muxCtxFunction); + } + this.lengthHistoryBuffer = lengthHistoryBuffer; + this.threadFactory = new WorkerThreadFactory(EventStore.class.getSimpleName() + "Worker", maxThreadNumber); + this.disruptor = new Disruptor<>(() -> new RingBufferEvent(filterConfig), ringBufferSize, threadFactory, isSingleProducer ? ProducerType.SINGLE : ProducerType.MULTI, waitStrategy); + final BiConsumer> clearCacheElement = (muxCtx, d) -> { + d.shutdown(); + final RingBuffer rb = d.getRingBuffer(); + for (long i = rb.getMinimumGatingSequence(); i < rb.getCursor(); i++) { + rb.get(i).clear(); + } + }; + this.eventStreams = muxBuilder == null ? Cache.>builder().withPostListener(clearCacheElement).build() : muxBuilder.build(); + + this.ctxMappingFunction = ctx -> { + // mux contexts -> create copy into separate disruptor/ringbuffer if necessary + // N.B. only single writer ... no further post-processors (all done in main eventStream) + final Disruptor ld = new Disruptor<>(() -> new RingBufferEvent(filterConfig), ringBufferSize, threadFactory, ProducerType.SINGLE, new BlockingWaitStrategy()); + ld.start(); + return ld; + }; + } + + public Disruptor getDisruptor() { + return disruptor; + } + + public List getHistory(final String muxCtx, final Predicate predicate, final int nHistory) { + return getHistory(muxCtx, predicate, Long.MAX_VALUE, nHistory); + } + + public List getHistory(final String muxCtx, final Predicate predicate, final long sequence, final int nHistory) { + assert muxCtx != null && !muxCtx.isBlank(); + assert sequence >= 0 : "sequence = " + sequence; + assert nHistory > 0 : "nHistory = " + nHistory; + final Disruptor localDisruptor = eventStreams.computeIfAbsent(muxCtx, ctxMappingFunction); + assert localDisruptor != null : NOT_FOUND_FOR_MULTIPLEXING_CONTEXT + muxCtx; + final RingBuffer ringBuffer = localDisruptor.getRingBuffer(); + + // simple consistency checks + final long cursor = ringBuffer.getCursor(); + assert cursor >= 0 : "uninitialised cursor: " + cursor; + assert nHistory < ringBuffer.getBufferSize() + : (" nHistory == " + nHistory + " history = new ArrayList<>(nHistory); + long seqStart = Math.max(cursor - ringBuffer.getBufferSize() - 1, 0); + for (long seq = cursor; history.size() < nHistory && seqStart <= seq; seq--) { + final RingBufferEvent evt = ringBuffer.get(seq); + if (evt.parentSequenceNumber <= sequence && predicate.test(evt)) { + history.add(evt); + } + } + return history; + } + + public Optional getLast(final String muxCtx, final Predicate predicate) { + return getLast(muxCtx, predicate, Long.MAX_VALUE); + } + + public Optional getLast(final String muxCtx, final Predicate predicate, final long sequence) { + assert muxCtx != null && !muxCtx.isBlank(); + final Disruptor localDisruptor = eventStreams.computeIfAbsent(muxCtx, ctxMappingFunction); + assert localDisruptor != null : NOT_FOUND_FOR_MULTIPLEXING_CONTEXT + muxCtx; + final RingBuffer ringBuffer = localDisruptor.getRingBuffer(); + assert ringBuffer.getCursor() > 0 : "uninitialised cursor: " + ringBuffer.getCursor(); + + // search for the most recent element that matches the provided predicate + long seqStart = Math.max(ringBuffer.getCursor() - ringBuffer.getBufferSize() - 1, 0); + for (long seq = ringBuffer.getCursor(); seqStart <= seq; seq--) { + final RingBufferEvent evt = ringBuffer.get(seq); + if (evt.parentSequenceNumber <= sequence && predicate.test(evt)) { + return Optional.of(evt.clone()); + } + } + return Optional.empty(); + } + + public RingBuffer getRingBuffer() { + return disruptor.getRingBuffer(); + } + + @SafeVarargs + public final LocalEventHandlerGroup register(final EventHandler... eventHandler) { + final LocalEventHandlerGroup group = new LocalEventHandlerGroup(lengthHistoryBuffer, eventHandler); + listener.add(group); + return group; + } + + public final LocalEventHandlerGroup register(final Predicate filter, Function muxCtxFunction, final HistoryEventHandler... eventHandler) { + final LocalEventHandlerGroup group = new LocalEventHandlerGroup(lengthHistoryBuffer, filter, muxCtxFunction, eventHandler); + listener.add(group); + return group; + } + + public void start(final boolean startReaper) { + // create single writer that is always executed first + EventHandler muxCtxWriter = (evt, seq, batch) -> { + for (Function muxCtxFunc : muxCtxFunctions) { + final String muxCtx = muxCtxFunc.apply(evt); + // only single writer ... no further post-processors (all done in main eventStream) + final Disruptor localDisruptor = eventStreams.computeIfAbsent(muxCtx, ctxMappingFunction); + assert localDisruptor != null : NOT_FOUND_FOR_MULTIPLEXING_CONTEXT + muxCtx; + + if (!localDisruptor.getRingBuffer().tryPublishEvent((event, sequence) -> { + if (event.payload != null && event.payload.getReferenceCount() > 0) { + event.payload.release(); + } + evt.copyTo(event); + })) { + throw new IllegalStateException("could not write event, sequence = " + seq + " muxCtx = " + muxCtx); + } + } + }; + allEventHandlers.add(muxCtxWriter); + EventHandlerGroup handlerGroup = disruptor.handleEventsWith(muxCtxWriter); + + // add other handler + for (LocalEventHandlerGroup localHandlerGroup : listener) { + attachHandler(disruptor, handlerGroup, localHandlerGroup); + } + + assert handlerGroup != null; + @SuppressWarnings("unchecked") + EventHandler[] eventHanders = allEventHandlers.toArray(new EventHandler[0]); + if (startReaper) { + // start the reaper thread for this given ring buffer + disruptor.after(eventHanders).then(new RingBufferEvent.ClearEventHandler()); + } + + // register this event store to all DefaultHistoryEventHandler + for (EventHandler handler : allEventHandlers) { + if (handler instanceof DefaultHistoryEventHandler) { + ((DefaultHistoryEventHandler) handler).setEventStore(this); + } + } + + disruptor.start(); + } + + public void start() { + this.start(true); + } + + public void stop() { + disruptor.shutdown(); + } + + protected EventHandlerGroup attachHandler(final Disruptor disruptor, final EventHandlerGroup parentGroup, final LocalEventHandlerGroup localHandlerGroup) { + EventHandlerGroup handlerGroup; + @SuppressWarnings("unchecked") + EventHandler[] eventHanders = localHandlerGroup.handler.toArray(new EventHandler[0]); + allEventHandlers.addAll(localHandlerGroup.handler); + if (parentGroup == null) { + handlerGroup = disruptor.handleEventsWith(eventHanders); + } else { + handlerGroup = parentGroup.then(eventHanders); + } + + if (localHandlerGroup.dependent != null && !localHandlerGroup.handler.isEmpty()) { + handlerGroup = attachHandler(disruptor, handlerGroup, localHandlerGroup.dependent); + } + + return handlerGroup; + } + + public static EventStoreFactory getFactory() { + return new EventStoreFactory(); + } + + public static class LocalEventHandlerGroup { + protected final List> handler = new NoDuplicatesList<>(); + protected final int lengthHistoryBuffer; + protected LocalEventHandlerGroup dependent; + + @SafeVarargs + private LocalEventHandlerGroup(final int lengthHistoryBuffer, final EventHandler... eventHandler) { + assert eventHandler != null; + this.lengthHistoryBuffer = lengthHistoryBuffer; + handler.addAll(Arrays.asList(eventHandler)); + } + + private LocalEventHandlerGroup(final int lengthHistoryBuffer, final Predicate filter, Function muxCtxFunction, final HistoryEventHandler... eventHandlerCallbacks) { + assert eventHandlerCallbacks != null; + this.lengthHistoryBuffer = lengthHistoryBuffer; + for (final HistoryEventHandler callback : eventHandlerCallbacks) { + handler.add(new DefaultHistoryEventHandler(null, filter, muxCtxFunction, lengthHistoryBuffer, callback)); // NOPMD - necessary to allocate inside loop + } + } + + @SafeVarargs + public final LocalEventHandlerGroup and(final EventHandler... eventHandler) { + assert eventHandler != null; + handler.addAll(Arrays.asList(eventHandler)); + return this; + } + + public final LocalEventHandlerGroup and(final Predicate filter, Function muxCtxFunction, final HistoryEventHandler... eventHandlerCallbacks) { + assert eventHandlerCallbacks != null; + for (final HistoryEventHandler callback : eventHandlerCallbacks) { + handler.add(new DefaultHistoryEventHandler(null, filter, muxCtxFunction, lengthHistoryBuffer, callback)); // NOPMD - necessary to allocate inside loop + } + return this; + } + + @SafeVarargs + public final LocalEventHandlerGroup then(final EventHandler... eventHandler) { + return (dependent = new LocalEventHandlerGroup(lengthHistoryBuffer, eventHandler)); // NOPMD NOSONAR + } + + public final LocalEventHandlerGroup then(final Predicate filter, Function muxCtxFunction, final HistoryEventHandler... eventHandlerCallbacks) { + return (dependent = new LocalEventHandlerGroup(lengthHistoryBuffer, filter, muxCtxFunction, eventHandlerCallbacks)); // NOPMD NOSONAR + } + } + + public static class EventStoreFactory { + private boolean singleProducer; + private int maxThreadNumber = 4; + private int ringbufferSize = 64; + private int lengthHistoryBuffer = 10; + private Cache.CacheBuilder> muxBuilder; + private Function muxCtxFunction; + private WaitStrategy waitStrategy = new TimeoutBlockingWaitStrategy(100, TimeUnit.MILLISECONDS); + @SuppressWarnings("unchecked") + private Class[] filterConfig = new Class[0]; + + public EventStore build() { + if (muxBuilder == null) { + muxBuilder = Cache.>builder().withLimit(lengthHistoryBuffer); + } + return new EventStore(muxBuilder, muxCtxFunction, ringbufferSize, lengthHistoryBuffer, maxThreadNumber, singleProducer, waitStrategy, filterConfig); + } + + public Class[] getFilterConfig() { + return filterConfig; + } + + @SafeVarargs + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + public final EventStoreFactory setFilterConfig(final Class... filterConfig) { + if (filterConfig == null) { + throw new IllegalArgumentException("filterConfig is null"); + } + this.filterConfig = filterConfig; + return this; + } + + public int getLengthHistoryBuffer() { + return lengthHistoryBuffer; + } + + public EventStoreFactory setLengthHistoryBuffer(final int lengthHistoryBuffer) { + if (lengthHistoryBuffer < 0) { + throw new IllegalArgumentException("lengthHistoryBuffer < 0: " + lengthHistoryBuffer); + } + + this.lengthHistoryBuffer = lengthHistoryBuffer; + return this; + } + + public int getMaxThreadNumber() { + return maxThreadNumber; + } + + public EventStoreFactory setMaxThreadNumber(final int maxThreadNumber) { + this.maxThreadNumber = maxThreadNumber; + return this; + } + + public Cache.CacheBuilder> getMuxBuilder() { + return muxBuilder; + } + + public EventStoreFactory setMuxBuilder(final Cache.CacheBuilder> muxBuilder) { + this.muxBuilder = muxBuilder; + return this; + } + + public Function getMuxCtxFunction() { + return muxCtxFunction; + } + + public EventStoreFactory setMuxCtxFunction(final Function muxCtxFunction) { + this.muxCtxFunction = muxCtxFunction; + return this; + } + + public int getRingbufferSize() { + return ringbufferSize; + } + + public EventStoreFactory setRingbufferSize(final int ringbufferSize) { + if (ringbufferSize < 0) { + throw new IllegalArgumentException("lengthHistoryBuffer < 0: " + ringbufferSize); + } + final int rounded = Util.ceilingNextPowerOfTwo(ringbufferSize - 1); + if (ringbufferSize != rounded) { + LOGGER.atWarn().addArgument(ringbufferSize).addArgument(rounded).log("setRingbufferSize({}) is not a power of two setting to next power of two: {}"); + this.ringbufferSize = rounded; + return this; + } + + this.ringbufferSize = ringbufferSize; + return this; + } + + public WaitStrategy getWaitStrategy() { + return waitStrategy; + } + + public EventStoreFactory setWaitStrategy(final WaitStrategy waitStrategy) { + this.waitStrategy = waitStrategy; + return this; + } + + public boolean isSingleProducer() { + return singleProducer; + } + + public EventStoreFactory setSingleProducer(final boolean singleProducer) { + this.singleProducer = singleProducer; + return this; + } + } + + protected static class DefaultHistoryEventHandler implements EventHandler { + private final Predicate filter; + private final Function muxCtxFunction; + private final HistoryEventHandler callback; + private final int lengthHistoryBuffer; + private EventStore eventStore; + private Cache> historyCache; + + protected DefaultHistoryEventHandler(final EventStore eventStore, final Predicate filter, Function muxCtxFunction, final int lengthHistoryBuffer, final HistoryEventHandler callback) { + assert filter != null : "filter predicate is null"; + assert muxCtxFunction != null : "muxCtxFunction hash function is null"; + assert callback != null : "callback function must not be null"; + + this.eventStore = eventStore; + this.filter = filter; + this.muxCtxFunction = muxCtxFunction; + this.lengthHistoryBuffer = lengthHistoryBuffer; + this.callback = callback; + } + + @Override + public void onEvent(final RingBufferEvent event, final long sequence, final boolean endOfBatch) { + if (!filter.test(event)) { + return; + } + final String muxCtx = muxCtxFunction.apply(event); + final LimitedArrayList history = historyCache.computeIfAbsent(muxCtx, ctx -> new LimitedArrayList<>(lengthHistoryBuffer)); + + final RingBufferEvent eventCopy = event.clone(); + if (history.size() == history.getLimit()) { + final RingBufferEvent removedEvent = history.remove(history.size() - 1); + removedEvent.clear(); + } + history.add(0, eventCopy); + + final RingBufferEvent result; + try { + result = callback.onEvent(history, eventStore, sequence, endOfBatch); + } catch (Exception e) { // NOPMD - part of exception handling/forwarding scheme + LOGGER.atError().setCause(e).addArgument(history.size()).addArgument(sequence).addArgument(endOfBatch) // + .log("caught error for arguments (history={}, eventStore, sequence={}, endOfBatch={})"); + event.throwables.add(e); + return; + } + if (result == null) { + return; + } + eventStore.getRingBuffer().publishEvent((newEvent, newSequence) -> { + result.copyTo(newEvent); + newEvent.parentSequenceNumber = newSequence; + }); + } + + private void setEventStore(final EventStore eventStore) { + this.eventStore = eventStore; + // allocate cache + final BiConsumer> clearCacheElement = (muxCtx, history) -> history.forEach(RingBufferEvent::clear); + final Cache c = eventStore.eventStreams; // re-use existing config limits + historyCache = Cache.>builder().withLimit((int) c.getLimit()).withTimeout(c.getTimeout(), c.getTimeUnit()).withPostListener(clearCacheElement).build(); + } + } +} diff --git a/core/src/main/java/io/opencmw/Filter.java b/core/src/main/java/io/opencmw/Filter.java new file mode 100644 index 00000000..89c96216 --- /dev/null +++ b/core/src/main/java/io/opencmw/Filter.java @@ -0,0 +1,31 @@ +package io.opencmw; + +/** + * Basic filter interface description + * + * @author rstein + * N.B. while 'toString()', 'hashCode()' and 'equals()' is ubiquously defined via the Java 'Object' class, these definition are kept for symmetry with the C++ implementation + */ +public interface Filter { + /** + * reinitialises the filter to safe default values + */ + void clear(); + + /** + * @param other filter this filter should copy its data to + */ + void copyTo(Filter other); + + @Override + boolean equals(Object other); + + @Override + int hashCode(); + + /** + * @return filter description including internal state (if any). + */ + @Override + String toString(); +} diff --git a/core/src/main/java/io/opencmw/FilterPredicate.java b/core/src/main/java/io/opencmw/FilterPredicate.java new file mode 100644 index 00000000..c9dee56c --- /dev/null +++ b/core/src/main/java/io/opencmw/FilterPredicate.java @@ -0,0 +1,40 @@ +package io.opencmw; + +import java.util.function.Predicate; + +//@FunctionalInterface +public interface FilterPredicate { + /** + * Evaluates this predicate on the given arguments. + * + * @param filterClass the filter class + * @param filterPredicate the filter predicate object + * @return {@code true} if the input arguments match the predicate, otherwise {@code false} + */ + boolean test(Class filterClass, Predicate filterPredicate); + + // /** + // * @param other a filter predicate that will be logically-ANDed with this predicate + // * @return a composed predicate that represents the short-circuiting logical AND of this predicate and the {@code other} predicate + // */ + // FilterPredicate and(FilterPredicate other); + + // /** + // * Returns a predicate that represents the logical negation of this + // * predicate. + // * + // * @return a predicate that represents the logical negation of this predicate + // */ + // FilterPredicate negate(); + + // + // /** + // * @param other a predicate that will be logically-ORed with this predicate + // * @return a composed predicate that represents the short-circuiting logical OR of this predicate and the {@code other} predicate + // * @throws NullPointerException if other is null + // */ + // default FilterPredicate or(FilterPredicate other) { + // Objects.requireNonNull(other); + // return (t, u) -> test(t, u) || other.test(t, u); + // } +} \ No newline at end of file diff --git a/core/src/main/java/io/opencmw/HistoryEventHandler.java b/core/src/main/java/io/opencmw/HistoryEventHandler.java new file mode 100644 index 00000000..0fb440f7 --- /dev/null +++ b/core/src/main/java/io/opencmw/HistoryEventHandler.java @@ -0,0 +1,23 @@ +package io.opencmw; + +import java.util.List; + +import com.lmax.disruptor.RingBuffer; + +@SuppressWarnings("PMD.SignatureDeclareThrowsException") // nature of this interface that it may and can throw any exception that needs to be dealt with upstream +public interface HistoryEventHandler { + /** + * Called when a publisher has published a new event to the {@link EventStore}. + * + * N.B. this is a delegate handler based on the {@link com.lmax.disruptor.EventHandler}. + * + * @param events RingBufferEvent history published to the {@link EventStore}. Newest element is stored in '0' + * @param eventStore handler to superordinate {@link EventStore} and {@link RingBuffer} + * @param sequence of the event being processed + * @param endOfBatch flag to indicate if this is the last event in a batch from the {@link EventStore} + * @return optional return element that publishes (if non-null) the new processed event in to the primary event stream + * + * @throws Exception if the EventHandler would like the exception handled further up the chain. (N.B. no further event is being published) + */ + RingBufferEvent onEvent(final List events, final EventStore eventStore, final long sequence, final boolean endOfBatch) throws Exception; +} diff --git a/core/src/main/java/io/opencmw/MimeType.java b/core/src/main/java/io/opencmw/MimeType.java new file mode 100644 index 00000000..00571a8d --- /dev/null +++ b/core/src/main/java/io/opencmw/MimeType.java @@ -0,0 +1,242 @@ +package io.opencmw; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import com.jsoniter.spi.JsoniterSpi; + +/** + * Definition and convenience methods for common MIME types according to RFC6838 and RFC4855 + *

+ * Since the official list is rather long, contains types we likely never encounter, and also does not contain all + * unofficial but nevertheless commonly used MIME types, we chose the specific sub-selection from: + * https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types + * + * @author rstein + * + */ +public enum MimeType { + /* text MIME types */ + CSS("text/css", "Cascading Style Sheets (CSS)", ".css"), + CSV("text/csv", "Comma-separated values (CSV)", ".csv"), + EVENT_STREAM("text/event-stream", "SSE stream"), + HTML("text/html", "HyperText Markup Language (HTML)", ".htm", ".html"), + ICS("text/calendar", "iCalendar format", ".ics"), + JAVASCRIPT("text/javascript", "JavaScript", ".js", ".mjs"), + JSON("application/json", "JSON format", ".json"), + JSON_LD("application/ld+json", "JSON-LD format", ".jsonld"), + TEXT("text/plain", "Text, (generally ASCII or ISO 8859-n)", ".txt"), + XML("text/xml", "XML", ".xml"), // if readable from casual users (RFC 3023, section 3) + YAML("text/yaml", "YAML Ain't Markup Language File", ".yml", ".yaml"), // not yet an IANA standard + + /* audio MIME types */ + AAC("audio/aac", "AAC audio", ".aac"), + MIDI("audio/midi", "Musical Instrument Digital Interface (MIDI)", ".mid", ".midi"), + MP3("audio/mpeg", "MP3 audio", ".mp3"), + OTF("audio/opus", "Opus audio", ".opus"), + WAV("audio/wav", "Waveform Audio Format", ".wav"), + WEBM_AUDIO("audio/webm", "WEBM audio", ".weba"), + + /* image MIME types */ + BMP("image/bmp", "Windows OS/2 Bitmap Graphics", ".bmp"), + GIF("image/gif", "Graphics Interchange Format (GIF)", ".gif"), + ICO("image/vnd.microsoft.icon", "Icon format", ".ico"), + JPEG("image/jpeg", "JPEG images", ".jpg", ".jpeg"), + PNG("image/png", "Portable Network Graphics", ".png"), + APNG("image/apng", "Portable Network Graphics", ".png", ".apng"), + SVG("image/svg+xml", "Scalable Vector Graphics (SVG)", ".svg"), + TIFF("image/tiff", "Tagged Image File Format (TIFF)", ".tif", ".tiff"), + WEBP("image/webp", "WEBP image", ".webp"), + + /* video MIME types */ + AVI("video/x-msvideo", "AVI: Audio Video Interleave", ".avi"), + MP2T("video/mp2t", "MPEG transport stream", ".ts"), + MPEG("video/mpeg", "MPEG Video", ".mpeg"), + WEBM_VIDEO("video/webm", "WEBM video", ".webm"), + + /* application-specific audio MIME types -- mostly binary-type formats */ + BINARY("application/octet-stream", "Any kind of binary data", ".bin"), + CMWLIGHT("application/cmwlight", "proprietary CERN serialiser binary format", ".cmwlight"), // deprecated: do not use for new projects + // BZIP("application/x-bzip", "BZip archive", ".bz"), // affected by patent + BZIP2("application/x-bzip2", "BZip2 archive", ".bz2"), + DOC("application/msword", "Microsoft Word", ".doc"), + DOCX("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "Microsoft Word (OpenXML)", ".docx"), + GZIP("application/gzip", "GZip Compressed Archive", ".gz"), + JAR("application/java-archive", "Java Archive (JAR)", ".jar"), + ODP("application/vnd.oasis.opendocument.presentation", "OpenDocument presentation document", ".odp"), + ODS("application/vnd.oasis.opendocument.spreadsheet", "OpenDocument spreadsheet document", ".ods"), + ODT("application/vnd.oasis.opendocument.text", "OpenDocument text document", ".odt"), + OGG("application/ogg", "OGG Audio/Video File", ".ogx", ".ogv", ".oga"), + PDF("application/pdf", "Adobe Portable Document Format (PDF)", ".pdf"), + PHP("application/x-httpd-php", "Hypertext Preprocessor (Personal Home Page)", ".php"), + PPT("application/vnd.ms-powerpoint", "Microsoft PowerPoint", ".ppt"), + PPTX("application/vnd.openxmlformats-officedocument.presentationml.presentation", "Microsoft PowerPoint (OpenXML)", ".pptx"), + RAR("application/vnd.rar", "RAR archive", ".rar"), + RTF("application/rtf", "Rich Text Format (RTF)", ".rtf"), + TAR("application/x-tar", "Tape Archive (TAR)", ".tar"), + VSD("application/vnd.visio", "Microsoft Visio", ".vsd"), + XHTML("application/xhtml+xml", "XHTML", ".xhtml"), + XLS("application/vnd.ms-excel", "Microsoft Excel", ".xls"), + XLSX("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "Microsoft Excel (OpenXML)", ".xlsx"), + ZIP("application/zip", "ZIP archive", ".zip"), + + /* fall-back */ + UNKNOWN("unknown/unknown", "unknown data format"); + + static { + // custom decoder to bypass Jsoniter's Javaassist usage that uses + // 'toString()' rather than 'name()' to instantiate specific enum values + JsoniterSpi.registerTypeDecoder(MimeType.class, iter -> MimeType.getEnum(iter.readString())); + } + + private final String mediaType; + private final String description; + private final Type type; + private final String subType; + private final List fileEndings; + + MimeType(final String definition, final String description, final String... endings) { + mediaType = definition; + this.description = description; + type = Type.getEnum(definition); + subType = definition.split("/")[1]; + fileEndings = Arrays.asList(endings); + } + + /** + * @return the commonly defined file-endings for the given MIME type + */ + public List getFileEndings() { + return fileEndings; + } + + /** + * @return the specific media sub-type, such as "plain" or "png", "mpeg", "mp4" + * or "xml". + */ + public String getSubType() { + return subType; + } + + /** + * @return the high-level media type, such as "text", "image", "audio", "video", + * or "application". + */ + public Type getType() { + return type; + } + + public boolean isImageData() { + return Type.IMAGE.equals(this.getType()); + } + + public boolean isNonDisplayableData() { + return !isImageData() && !isVideoData(); + } + + public boolean isTextData() { + return Type.TEXT.equals(this.getType()); + } + + public boolean isVideoData() { + return Type.VIDEO.equals(this.getType()); + } + + @Override + public String toString() { + return mediaType; + } + + public String getMediaType() { + return mediaType; + } + + /** + * @return human-readable description of the format + */ + public String getDescription() { + return description; + } + + /** + * Case-insensitive mapping between MIME-type string and enumumeration value. + * + * @param mimeType the string equivalent mime-type, e.g. "image/png" + * @return the enumeration equivalent first matching mime-type, e.g. MimeType.PNG or MimeType.UNKNOWN as fall-back + */ + public static MimeType getEnum(final String mimeType) { + if (mimeType == null || mimeType.isBlank()) { + return UNKNOWN; + } + + final String trimmed = mimeType.toLowerCase(Locale.UK).trim(); + for (MimeType mType : MimeType.values()) { + // N.B.trimmed can contain several MIME types, e.g "image/webp,image/apng,image/*" + if (trimmed.contains(mType.mediaType)) { + return mType; + } + // second fall-back - raw enum type name + if (trimmed.equalsIgnoreCase(mType.name())) { + return mType; + } + } + return UNKNOWN; + } + + /** + * Case-insensitive mapping between MIME-type string and enumeration value. + * + * @param fileName the string equivalent mime-type, e.g. "image/png" + * @return the enumeration equivalent mime-type, e.g. MimeType.PNG or MimeType.UNKNOWN as fall-back + */ + public static MimeType getEnumByFileName(final String fileName) { + if (fileName == null || fileName.isBlank()) { + return UNKNOWN; + } + + final String trimmed = fileName.toLowerCase(Locale.UK).trim(); + for (MimeType mType : MimeType.values()) { + for (String ending : mType.getFileEndings()) { + if (trimmed.endsWith(ending)) { + return mType; + } + } + } + + return UNKNOWN; + } + + public enum Type { + AUDIO("audio"), + IMAGE("image"), + VIDEO("video"), + TEXT("text"), + APPLICATION("application"), + UNKNOWN("unknown"); + + private final String typeDef; + + Type(final String subType) { + typeDef = subType; + } + + @Override + public String toString() { + return typeDef; + } + + public static Type getEnum(final String type) { + if (type == null || type.isBlank()) { + return UNKNOWN; + } + final String stripped = type.split("/")[0]; + for (Type mSubType : Type.values()) { + if (mSubType.typeDef.equalsIgnoreCase(stripped)) { + return mSubType; + } + } + return UNKNOWN; + } + } +} diff --git a/core/src/main/java/io/opencmw/OpenCmwProtocol.java b/core/src/main/java/io/opencmw/OpenCmwProtocol.java new file mode 100644 index 00000000..6ae5dbd0 --- /dev/null +++ b/core/src/main/java/io/opencmw/OpenCmwProtocol.java @@ -0,0 +1,526 @@ +package io.opencmw; + +import static org.zeromq.ZMQ.Socket; + +import static io.opencmw.OpenCmwProtocol.Command.*; +import static io.opencmw.utils.AnsiDefs.ANSI_RED; +import static io.opencmw.utils.AnsiDefs.ANSI_RESET; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZFrame; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; +import org.zeromq.util.ZData; + +import io.opencmw.utils.AnsiDefs; + +/** + * Open Common Middle-Ware Protocol + * + * extended and based upon: + * Majordomo Protocol (MDP) definitions and implementations according to https://rfc.zeromq.org/spec/7/ + * + * For a non-programmatic protocol description see: + * https://github.com/GSI-CS-CO/chart-fx/blob/master/microservice/docs/MajordomoProtocol.md + * + * @author rstein + * @author Alexander Krimm + */ +@SuppressWarnings({ "PMD.TooManyMethods", "PMD.ArrayIsStoredDirectly", "PMD.CommentSize", "PMD.MethodReturnsInternalArray" }) +public final class OpenCmwProtocol { // NOPMD - nomen est omen + public static final String COMMAND_MUST_NOT_BE_NULL = "command must not be null"; + public static final byte[] EMPTY_FRAME = {}; + public static final URI EMPTY_URI = URI.create(""); + private static final byte[] PROTOCOL_NAME_CLIENT = "MDPC03".getBytes(StandardCharsets.UTF_8); + private static final byte[] PROTOCOL_NAME_CLIENT_HTTP = "MDPH03".getBytes(StandardCharsets.UTF_8); + private static final byte[] PROTOCOL_NAME_WORKER = "MDPW03".getBytes(StandardCharsets.UTF_8); + private static final byte[] PROTOCOL_NAME_UNKNOWN = "UNKNOWN_PROTOCOL".getBytes(StandardCharsets.UTF_8); + private static final int MAX_PRINT_LENGTH = 200; // unique client id, see ROUTER socket docs for info + private static final int FRAME0_SOURCE_ID = 0; // unique client id, see ROUTER socket docs for info + private static final int FRAME1_PROTOCOL_ID = 1; // 'MDPC0' or 'MDPW0' + private static final int FRAME2_COMMAND_ID = 2; + private static final int FRAME3_SERVICE_ID = 3; + private static final int FRAME4_CLIENT_REQUEST_ID = 4; + private static final int FRAME5_TOPIC = 5; + private static final int FRAME6_DATA = 6; + private static final int FRAME7_ERROR = 7; + private static final int FRAME8_RBAC_TOKEN = 8; + private static final Logger LOGGER = LoggerFactory.getLogger(OpenCmwProtocol.class); + private static final String SOCKET_MUST_NOT_BE_NULL = "socket must not be null"; + public static final int N_PROTOCOL_FRAMES = 8; + + /** + * MDP sub-protocol V0.1 + */ + public enum MdpSubProtocol { + PROT_CLIENT(PROTOCOL_NAME_CLIENT), // OpenCmwProtocol/Client protocol implementation version + PROT_CLIENT_HTTP(PROTOCOL_NAME_CLIENT_HTTP), // OpenCmwProtocol/HTTP(REST) protocol implementation version + PROT_WORKER(PROTOCOL_NAME_WORKER), // OpenCmwProtocol/Worker protocol implementation version + UNKNOWN(PROTOCOL_NAME_UNKNOWN); + + private final byte[] data; + private final String protocolName; + MdpSubProtocol(final byte[] value) { + this.data = value; + protocolName = new String(data, StandardCharsets.UTF_8); + } + + public byte[] getData() { + return data; + } + + @Override + public String toString() { + return "MdpSubProtocol{'" + protocolName + "'}"; + } + + public static MdpSubProtocol getProtocol(byte[] frame) { + for (MdpSubProtocol knownProtocol : MdpSubProtocol.values()) { + if (Arrays.equals(knownProtocol.data, frame)) { + if (knownProtocol == UNKNOWN) { + continue; + } + return knownProtocol; + } + } + return UNKNOWN; + } + } + + /** + * OpenCmwProtocol commands, as byte values + */ + public enum Command { + GET_REQUEST(0x01, true, true), + SET_REQUEST(0x02, true, true), + PARTIAL(0x03, true, true), + FINAL(0x04, true, true), + READY(0x05, true, true), // mandatory for worker, optional for client (ie. for optional initial RBAC authentication) + DISCONNECT(0x06, true, true), // mandatory for worker, optional for client + SUBSCRIBE(0x07, true, true), // client specific command + UNSUBSCRIBE(0x08, true, true), // client specific command + W_NOTIFY(0x09, false, true), // worker specific command + W_HEARTBEAT(0x10, true, true), // worker specific command, optional for client + UNKNOWN(-1, false, false); + + private final byte[] data; + private final boolean isForClients; + private final boolean isForWorkers; + Command(final int value, boolean client, final boolean worker) { //watch for ints>255, will be truncated + this.data = new byte[] { (byte) (value & 0xFF) }; + this.isForClients = client; + this.isForWorkers = worker; + } + + public byte[] getData() { + return data; + } + + public boolean isClientCompatible() { + return isForClients; + } + + public boolean isWorkerCompatible() { + return isForWorkers; + } + + public static Command getCommand(byte[] frame) { + for (Command knownMdpCommand : values()) { + if (Arrays.equals(knownMdpCommand.data, frame)) { + if (knownMdpCommand == UNKNOWN) { + continue; + } + return knownMdpCommand; + } + } + return UNKNOWN; + } + } + + /** + * MDP data object to store OpenCMW frames description + * + * For a non-programmatic protocol description see: + * https://github.com/GSI-CS-CO/chart-fx/blob/master/microservice/docs/MajordomoProtocol.md + */ + public static class MdpMessage { + /** OpenCMW frame 0: sender source ID - usually the ID from the MDP broker ROUTER socket for the given connection */ + public byte[] senderID; + /** OpenCMW frame 1: unique protocol identifier */ + public MdpSubProtocol protocol; + /** OpenCMW frame 2: MDP command */ + public Command command; + /** OpenCMW frame 3: service name (for client sub-protocols) or client source ID (for worker sub-protocol) */ + public byte[] serviceNameBytes; // UTF-8 encoded service name and/or clientID + /** OpenCMW frame 4: custom client request ID (N.B. client-generated and transparently passed through broker and worker) */ + public byte[] clientRequestID; + /** OpenCMW frame 5: request/reply topic -- follows URI syntax, ie. '

scheme:[//authority]path[?query][#fragment]
' see documentation */ + public URI topic; // request/reply topic - follows URI syntax, ie. '
scheme:[//authority]path[?query][#fragment]
' + /** OpenCMW frame 6: data (may be null if error stack is not blank) */ + public byte[] data; + /** OpenCMW frame 7: error stack -- UTF-8 string (may be blank if data is not null) */ + public String errors; + /** OpenCMW frame 8 (optional): RBAC token */ + public byte[] rbacToken; + + private MdpMessage() { + // private constructor + } + + /** + * generate new (immutable) MdpMessage representation + * @param senderID OpenCMW frame 0: sender source ID - usually the ID from the MDP broker ROUTER socket for the given connection + * @param protocol OpenCMW frame 1: unique protocol identifier (see: MdpSubProtocol) + * @param command OpenCMW frame 2: command (see: Command) + * @param serviceID OpenCMW frame 3: service name (for client sub-protocols) or client source ID (for worker sub-protocol) + * @param clientRequestID OpenCMW frame 4: custom client request ID (N.B. client-generated and transparently passed through broker and worker) + * @param topic openCMW frame 5: the request/reply topic - follows URI syntax, ie. '
scheme:[//authority]path[?query][#fragment]
' see documentation + * @param data OpenCMW frame 6: data - may be null in case errors is not null + * @param errors OpenCMW frame 7: error stack -- UTF-8 string may be blank only if data is not null + * @param rbacToken OpenCMW frame 8 (optional): RBAC token + */ + public MdpMessage(final byte[] senderID, @NotNull final MdpSubProtocol protocol, @NotNull final Command command, + @NotNull final byte[] serviceID, @NotNull final byte[] clientRequestID, @NotNull final URI topic, + final byte[] data, @NotNull final String errors, final byte[] rbacToken) { + this.senderID = senderID == null ? EMPTY_FRAME : senderID; + this.protocol = protocol; + this.command = command; + this.serviceNameBytes = serviceID; + this.clientRequestID = clientRequestID; + this.topic = topic; + this.data = data == null ? EMPTY_FRAME : data; + this.errors = errors; + if (data == null && errors.isBlank()) { + throw new IllegalArgumentException("data must not be null if errors are blank"); + } + this.rbacToken = rbacToken == null ? EMPTY_FRAME : rbacToken; + } + + /** + * Copy constructor cloning other MdpMessage + * @param other MdpMessage + */ + public MdpMessage(@NotNull final MdpMessage other) { + this(other, UNKNOWN); + } + + /** + * Copy constructor cloning other MdpMessage + * @param other MdpMessage + * @param fullCopy is UNKNOWN then a clone is generated, for all other cases a MdpMessage with + * the specified command, mirrored frames, except 'data', 'errors' and 'rbacToken' is generated. + */ + public MdpMessage(@NotNull final MdpMessage other, @NotNull final Command fullCopy) { + this(copyOf(other.senderID), other.protocol, fullCopy == UNKNOWN ? other.command : fullCopy, copyOf(other.serviceNameBytes), copyOf(other.clientRequestID), other.topic, + fullCopy == UNKNOWN ? copyOf(other.data) : EMPTY_FRAME, fullCopy == UNKNOWN ? other.errors : "", fullCopy == UNKNOWN ? copyOf(other.rbacToken) : EMPTY_FRAME); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } else if (!(obj instanceof MdpMessage)) { + return false; + } + final MdpMessage other = (MdpMessage) obj; + // ignore senderID from comparison since not all socket provide/require this information + if (/*!Arrays.equals(senderID, other.senderID) ||*/ (protocol != other.protocol) || (command != other.command) + || !Arrays.equals(serviceNameBytes, other.serviceNameBytes) || !Arrays.equals(clientRequestID, other.clientRequestID) + || (!Objects.equals(topic, other.topic)) + || !Arrays.equals(data, other.data) + || (!Objects.equals(errors, other.errors))) { + return false; + } + return Arrays.equals(rbacToken, other.rbacToken); + } + + public String getSenderName() { + return senderID == null ? "" : ZData.toString(senderID); + } + + public String getServiceName() { + return serviceNameBytes == null ? "" : ZData.toString(serviceNameBytes); + } + + public boolean hasRbackToken() { + return rbacToken.length != 0; + } + + @Override + public int hashCode() { + int result = (protocol == null ? 0 : protocol.hashCode()); + result = 31 * result + Objects.hashCode(command); + result = 31 * result + Arrays.hashCode(serviceNameBytes); + result = 31 * result + Arrays.hashCode(clientRequestID); + result = 31 * result + (topic == null ? 0 : topic.hashCode()); + result = 31 * result + Arrays.hashCode(data); + result = 31 * result + (errors == null ? 0 : errors.hashCode()); + result = 31 * result + Arrays.hashCode(rbacToken); + return result; + } + + /** + * Send MDP message to Socket + * + * @param socket ZeroMQ socket to send the message on + * @return {@code true} if successful + */ + public boolean send(final Socket socket) { + // some assertions for debugging purposes - N.B. these should occur only when developing/refactoring the frame-work + // N.B. to be enabled with '-ea' VM argument + assert socket != null : SOCKET_MUST_NOT_BE_NULL; + assert protocol != null : "protocol must not be null"; + assert !protocol.equals(MdpSubProtocol.UNKNOWN) + : "protocol must not be UNKNOWN"; + assert command != null : COMMAND_MUST_NOT_BE_NULL; + assert (protocol.equals(MdpSubProtocol.PROT_CLIENT) && command.isClientCompatible()) + || (protocol.equals(MdpSubProtocol.PROT_WORKER) && command.isWorkerCompatible()) + : "command is client/worker compatible"; + assert serviceNameBytes != null : "serviceName must not be null"; + assert clientRequestID != null : "clientRequestID must not be null"; + assert topic != null : "topic must not be null"; + assert !(data == null && (errors == null || errors.isBlank())) + : "data must not be null and errors be blank"; + + ZMsg msg = new ZMsg(); + if (socket.getSocketType() == SocketType.ROUTER) { + if (senderID == null) { + throw new IllegalArgumentException("senderID must be non-null when using ROUTER sockets"); + } + msg.add(new ZFrame(senderID)); // frame 0: source ID (optional, only needed for broker sockets) + } + msg.add(new ZFrame(protocol.data)); // frame: 1 + msg.add(new ZFrame(command.data)); // frame: 2 + msg.add(new ZFrame(serviceNameBytes)); // frame: 3 + msg.add(new ZFrame(clientRequestID)); // frame: 4 + msg.addString(topic.toString()); // frame: 5 + msg.add(new ZFrame(data == null ? EMPTY_FRAME : data)); // frame: 6 + msg.addString(errors == null ? "" : errors); // frame: 7 + msg.add(new ZFrame(rbacToken)); // frame: 8 - rbac token + + if (LOGGER.isTraceEnabled()) { + LOGGER.atTrace().addArgument(msg.toString()).log("sending message {}"); + } + return msg.send(socket); + } + + @Override + public String toString() { + final String errStr = errors == null || errors.isBlank() ? "no-exception" : ANSI_RED + " exception thrown: " + errors + ANSI_RESET; + return "MdpMessage{senderID='" + ZData.toString(senderID) + "', " + protocol + ", " + command + ", serviceName='" + getServiceName() + + "', clientRequestID='" + ZData.toString(clientRequestID) + "', topic='" + topic + + "', data='" + dataToString(data) + "', " + errStr + ", rbac='" + ZData.toString(rbacToken) + "'}"; + } + + protected static String dataToString(byte[] data) { + if (data == null) { + return ""; + } + // Dump message as text or hex-encoded string + boolean isText = true; + for (byte aData : data) { + if (aData < AnsiDefs.MIN_PRINTABLE_CHAR) { + isText = false; + break; + } + } + if (isText) { + // always make full-print when there are only printable characters + return new String(data, ZMQ.CHARSET); + } + if (data.length < MAX_PRINT_LENGTH) { + return ZData.strhex(data); + } else { + return ZData.strhex(Arrays.copyOf(data, MAX_PRINT_LENGTH)) + "[" + (data.length - MAX_PRINT_LENGTH) + " more bytes]"; + } + } + + /** + * @param socket the socket to receive from (performs blocking call) + * @return MdpMessage if valid, or {@code null} otherwise + */ + public static MdpMessage receive(final Socket socket) { + return receive(socket, true); + } + + /** + * @param socket the socket to receive from + * @param wait setting the flag to ZMQ.DONTWAIT does a non-blocking recv. + * @return MdpMessage if valid, or {@code null} otherwise + */ + @SuppressWarnings("PMD.NPathComplexity") + public static MdpMessage receive(@NotNull final Socket socket, final boolean wait) { + final int flags = wait ? 0 : ZMQ.DONTWAIT; + final ZMsg msg = ZMsg.recvMsg(socket, flags); + if (msg == null) { + return null; + } + if (socket.getSocketType() != SocketType.ROUTER) { + msg.push(EMPTY_FRAME); // push empty client frame + } + + if (socket.getSocketType() == SocketType.SUB || socket.getSocketType() == SocketType.XSUB) { + msg.pollFirst(); // remove first subscription topic message -- not needed here since this is also encoded in the service/topic frame + } + + final List rawFrames = msg.stream().map(ZFrame::getData).collect(Collectors.toUnmodifiableList()); + if (rawFrames.size() <= N_PROTOCOL_FRAMES) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(rawFrames.size()).addArgument(dataToString(rawFrames)).log("received message size is < " + N_PROTOCOL_FRAMES + ": {} rawMessage: {}"); + } + return null; + } + + // OpenCMW frame 0: sender source ID - usually the ID from the MDP broker ROUTER socket for the given connection + final byte[] senderID = socket.getSocketType() == SocketType.ROUTER ? rawFrames.get(FRAME0_SOURCE_ID) : EMPTY_FRAME; // NOPMD + // OpenCMW frame 1: unique protocol identifier + final MdpSubProtocol protocol = MdpSubProtocol.getProtocol(rawFrames.get(FRAME1_PROTOCOL_ID)); + if (protocol.equals(MdpSubProtocol.UNKNOWN)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(ZData.toString(rawFrames.get(FRAME1_PROTOCOL_ID))).addArgument(dataToString(rawFrames)).log("unknown protocol: '{}' rawMessage: {}"); + } + return null; + } + + // OpenCMW frame 2: command + final Command command = getCommand(rawFrames.get(FRAME2_COMMAND_ID)); + if (command.equals(UNKNOWN)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(ZData.toString(rawFrames.get(FRAME2_COMMAND_ID))).addArgument(dataToString(rawFrames)).log("unknown command: '{}' rawMessage: {}"); + } + return null; + } + + // OpenCMW frame 3: service name or client source ID + final byte[] serviceNameBytes = rawFrames.get(FRAME3_SERVICE_ID); + if (serviceNameBytes == null) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addKeyValue("a", "dd").log(""); + LOGGER.atWarn().addArgument(dataToString(rawFrames)).log("serviceNameBytes is null, rawMessage: {}"); + } + return null; + } + + // OpenCMW frame 4: service name or client source ID + final byte[] clientRequestID = rawFrames.get(FRAME4_CLIENT_REQUEST_ID); // NOPMD + + // OpenCMW frame 5: request/reply topic -- UTF-8 string + final byte[] topicBytes = rawFrames.get(FRAME5_TOPIC); + if (topicBytes == null) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(dataToString(rawFrames)).log("topic is null, rawMessage: {}"); + } + return null; + } + + final String topicString = new String(topicBytes, StandardCharsets.UTF_8); + final URI topic; + try { + topic = new URI(topicString); + } catch (URISyntaxException e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().setCause(e).addArgument(topicString).addArgument(topicString).log("topic URI cannot be parsed {} - {}"); + } + return null; + } + + // OpenCMW frame 6: data + final byte[] data = rawFrames.get(FRAME6_DATA); + // OpenCMW frame 7: error stack -- UTF-8 string + final byte[] errorBytes = rawFrames.get(FRAME7_ERROR); + final String errors = errorBytes == null || errorBytes.length == 0 ? "" : new String(errorBytes, StandardCharsets.UTF_8); + if (data == null && errors.isBlank()) { + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(dataToString(rawFrames)).log("data is null and errors is blank - {}"); + } + return null; + } + + // OpenCMW frame 8 (optional): RBAC token + final byte[] rbacTokenByte = rawFrames.size() == 9 ? rawFrames.get(FRAME8_RBAC_TOKEN) : null; + final byte[] rbacToken = rawFrames.size() == 9 && rbacTokenByte != null ? rbacTokenByte : EMPTY_FRAME; + + return new MdpMessage(senderID, protocol, command, serviceNameBytes, clientRequestID, topic, data, errors, rbacToken); // OpenCMW frame 8 (optional): RBAC token + } + + public static boolean send(final Socket socket, final List replies) { + assert socket != null : SOCKET_MUST_NOT_BE_NULL; + assert replies != null; + if (replies.isEmpty()) { + return false; + } + boolean sendState = false; + for (Iterator iter = replies.iterator(); iter.hasNext();) { + MdpMessage reply = iter.next(); + reply.command = iter.hasNext() ? PARTIAL : FINAL; + sendState |= reply.send(socket); + } + return sendState; + } + + protected static byte[] copyOf(byte[] original) { + return original == null ? EMPTY_FRAME : Arrays.copyOf(original, original.length); + } + + protected static String dataToString(List data) { + return data.stream().map(ZData::toString).collect(Collectors.joining(", ", "[#frames= " + data.size() + ": ", "]")); + } + } + + /** + * MDP reply/request context + */ + public static class Context { + public final MdpMessage req; // input request + public MdpMessage rep; // return request + + private Context() { + req = new MdpMessage(); + } + + public Context(@NotNull MdpMessage requestMsg) { + req = requestMsg; + rep = new MdpMessage(req, FINAL); + } + + @Override + public String toString() { + return "OpenCmwProtocol.Context{req=" + req + ", rep=" + rep + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Context)) { + return false; + } + Context context = (Context) o; + + if (!Objects.equals(req, context.req)) { + return false; + } + return Objects.equals(rep, context.rep); + } + + @Override + public int hashCode() { + int result = req == null ? 0 : req.hashCode(); + result = 31 * result + (rep == null ? 0 : rep.hashCode()); + return result; + } + } +} diff --git a/core/src/main/java/io/opencmw/QueryParameterParser.java b/core/src/main/java/io/opencmw/QueryParameterParser.java new file mode 100644 index 00000000..3fc486f7 --- /dev/null +++ b/core/src/main/java/io/opencmw/QueryParameterParser.java @@ -0,0 +1,296 @@ +package io.opencmw; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.AbstractMap.SimpleImmutableEntry; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; + +import io.opencmw.filter.TimingCtx; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.spi.ClassFieldDescription; +import io.opencmw.serialiser.utils.ClassUtils; + +/** + * Parses query parameters into PoJo structure. + * + * Follows URI syntax, ie. '
scheme:[//authority]path[?query][#fragment]
' + * see documentation + * + * @author rstein + */ +public final class QueryParameterParser { // NOPMD - nomen est omen + public static final String MIME_TYPE_TAG = "contentType"; + public static final ConcurrentMap STRING_TO_CLASS_CONVERTER = new ConcurrentHashMap<>(); // NOSONAR NOPMD + public static final ConcurrentMap> CLASS_TO_STRING_CONVERTER = new ConcurrentHashMap<>(); // NOSONAR NOPMD + public static final ConcurrentMap> CLASS_TO_OBJECT_CONVERTER = new ConcurrentHashMap<>(); // NOSONAR NOPMD + static { + STRING_TO_CLASS_CONVERTER.put(boolean.class, (str, obj, field) -> field.getField().setBoolean(obj, Boolean.parseBoolean(str))); + STRING_TO_CLASS_CONVERTER.put(byte.class, (str, obj, field) -> field.getField().setByte(obj, Byte.parseByte(str))); + STRING_TO_CLASS_CONVERTER.put(short.class, (str, obj, field) -> field.getField().setShort(obj, Short.parseShort(str))); + STRING_TO_CLASS_CONVERTER.put(int.class, (str, obj, field) -> field.getField().setInt(obj, Integer.parseInt(str))); + STRING_TO_CLASS_CONVERTER.put(long.class, (str, obj, field) -> field.getField().setLong(obj, Long.parseLong(str))); + STRING_TO_CLASS_CONVERTER.put(float.class, (str, obj, field) -> field.getField().setFloat(obj, Float.parseFloat(str))); + STRING_TO_CLASS_CONVERTER.put(double.class, (str, obj, field) -> field.getField().setDouble(obj, Double.parseDouble(str))); + STRING_TO_CLASS_CONVERTER.put(Boolean.class, (str, obj, field) -> field.getField().set(obj, Boolean.parseBoolean(str))); + STRING_TO_CLASS_CONVERTER.put(Byte.class, (str, obj, field) -> field.getField().set(obj, Byte.parseByte(str))); + STRING_TO_CLASS_CONVERTER.put(Short.class, (str, obj, field) -> field.getField().set(obj, Short.parseShort(str))); + STRING_TO_CLASS_CONVERTER.put(Integer.class, (str, obj, field) -> field.getField().set(obj, Integer.parseInt(str))); + STRING_TO_CLASS_CONVERTER.put(Long.class, (str, obj, field) -> field.getField().set(obj, Long.parseLong(str))); + STRING_TO_CLASS_CONVERTER.put(Float.class, (str, obj, field) -> field.getField().set(obj, Float.parseFloat(str))); + STRING_TO_CLASS_CONVERTER.put(Double.class, (str, obj, field) -> field.getField().set(obj, Double.parseDouble(str))); + STRING_TO_CLASS_CONVERTER.put(String.class, (str, obj, field) -> field.getField().set(obj, str)); + + final BiFunction objToString = (obj, field) -> { + final Object ret = field.getField().get(obj); + return ret == null || ret.getClass().equals(Object.class) ? "" : ret.toString(); + }; + CLASS_TO_STRING_CONVERTER.put(boolean.class, (obj, field) -> Boolean.toString(field.getField().getBoolean(obj))); + CLASS_TO_STRING_CONVERTER.put(byte.class, (obj, field) -> Byte.toString(field.getField().getByte(obj))); + CLASS_TO_STRING_CONVERTER.put(short.class, (obj, field) -> Short.toString(field.getField().getShort(obj))); + CLASS_TO_STRING_CONVERTER.put(int.class, (obj, field) -> Integer.toString(field.getField().getInt(obj))); + CLASS_TO_STRING_CONVERTER.put(long.class, (obj, field) -> Long.toString(field.getField().getLong(obj))); + CLASS_TO_STRING_CONVERTER.put(float.class, (obj, field) -> Float.toString(field.getField().getFloat(obj))); + CLASS_TO_STRING_CONVERTER.put(double.class, (obj, field) -> Double.toString(field.getField().getDouble(obj))); + CLASS_TO_STRING_CONVERTER.put(boolean[].class, (obj, field) -> Arrays.toString((boolean[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(byte[].class, (obj, field) -> Arrays.toString((byte[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(short[].class, (obj, field) -> Arrays.toString((short[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(int[].class, (obj, field) -> Arrays.toString((int[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(long[].class, (obj, field) -> Arrays.toString((long[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(float[].class, (obj, field) -> Arrays.toString((float[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(double[].class, (obj, field) -> Arrays.toString((double[]) field.getField().get(obj))); + CLASS_TO_STRING_CONVERTER.put(Boolean.class, objToString); + CLASS_TO_STRING_CONVERTER.put(Byte.class, objToString); + CLASS_TO_STRING_CONVERTER.put(Short.class, objToString); + CLASS_TO_STRING_CONVERTER.put(Integer.class, objToString); + CLASS_TO_STRING_CONVERTER.put(Long.class, objToString); + CLASS_TO_STRING_CONVERTER.put(Float.class, objToString); + CLASS_TO_STRING_CONVERTER.put(Double.class, objToString); + CLASS_TO_STRING_CONVERTER.put(String.class, (obj, field) -> Objects.requireNonNullElse(field.getField().get(obj), "").toString()); + + CLASS_TO_OBJECT_CONVERTER.put(boolean.class, (obj, field) -> field.getField().getBoolean(obj)); + CLASS_TO_OBJECT_CONVERTER.put(byte.class, (obj, field) -> field.getField().getByte(obj)); + CLASS_TO_OBJECT_CONVERTER.put(short.class, (obj, field) -> field.getField().getShort(obj)); + CLASS_TO_OBJECT_CONVERTER.put(int.class, (obj, field) -> field.getField().getInt(obj)); + CLASS_TO_OBJECT_CONVERTER.put(long.class, (obj, field) -> field.getField().getLong(obj)); + CLASS_TO_OBJECT_CONVERTER.put(float.class, (obj, field) -> field.getField().getFloat(obj)); + CLASS_TO_OBJECT_CONVERTER.put(double.class, (obj, field) -> field.getField().getDouble(obj)); + CLASS_TO_OBJECT_CONVERTER.put(Object.class, (obj, field) -> field.getField().get(obj)); + + // special known objects + STRING_TO_CLASS_CONVERTER.put(Object.class, (str, obj, field) -> field.getField().set(obj, new Object())); + STRING_TO_CLASS_CONVERTER.put(MimeType.class, (str, obj, field) -> field.getField().set(obj, MimeType.getEnum(str))); + STRING_TO_CLASS_CONVERTER.put(TimingCtx.class, (str, obj, field) -> field.getField().set(obj, TimingCtx.get(str))); + + CLASS_TO_STRING_CONVERTER.put(Object.class, objToString); + CLASS_TO_STRING_CONVERTER.put(MimeType.class, (obj, field) -> { + final Object ret = field.getField().get(obj); + return ret == null || ret.getClass().equals(Object.class) ? "" : ((MimeType) ret).name(); + }); + CLASS_TO_STRING_CONVERTER.put(TimingCtx.class, (obj, field) -> { + final Object ctx = field.getField().get(obj); + return ctx instanceof TimingCtx ? ((TimingCtx) ctx).selector : ""; + }); + } + + private QueryParameterParser() { + // this is a utility class + } + + public static URI appendQueryParameter(URI oldUri, String appendQuery) throws URISyntaxException { + if (appendQuery == null || appendQuery.isBlank()) { + return oldUri; + } + return new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(), oldUri.getQuery() == null ? appendQuery : (oldUri.getQuery() + "&" + appendQuery), oldUri.getFragment()); + } + + /** + * + * @param queryParameterMap query parameter map + * @return queryString a rfc3986 query parameter string + */ + @SuppressWarnings("PMD") + public static String generateQueryParameter(final Map queryParameterMap) { //NOSONAR - complexity justified + final StringBuilder builder = new StringBuilder(); + + final Set> entrySet = queryParameterMap.entrySet(); + final Iterator> iterator = entrySet.iterator(); + boolean first = true; + while (iterator.hasNext()) { + Map.Entry item = iterator.next(); + String key = item.getKey(); + Object values = item.getValue(); + if (!first) { + builder.append('&'); + } + if (values == null) { + builder.append(key); + } else if (List.class.isAssignableFrom(values.getClass())) { + @SuppressWarnings("unchecked") // checked with above isAssignableFrom + List list = (List) values; + for (Object val : list) { + if (!first) { + builder.append('&'); + } + if (val == null) { + builder.append(key); + } else { + builder.append(key).append('=').append(val); + } + first = false; + } + } else { + // non list object + builder.append(key).append('=').append(values); + } + first = false; + } + return builder.toString(); + } + + /** + * + * @param obj storage class + * @return queryString a rfc3986 query parameter string + */ + public static String generateQueryParameter(Object obj) { + final ClassFieldDescription fieldDescription = ClassUtils.getFieldDescription(obj.getClass()); + final StringBuilder builder = new StringBuilder(); + final List children = fieldDescription.getChildren(); + for (int index = 0; index < children.size(); index++) { + ClassFieldDescription field = (ClassFieldDescription) children.get(index); + final BiFunction mapFunction = CLASS_TO_STRING_CONVERTER.get(field.getType()); + final String str; + if (mapFunction == null) { + str = CLASS_TO_STRING_CONVERTER.get(Object.class).apply(obj, field); + } else { + str = mapFunction.apply(obj, field); + } + builder.append(field.getFieldName()).append('=').append(str == null ? "" : URLEncoder.encode(str, UTF_8)); + if (index != children.size() - 1) { + builder.append('&'); + } + } + return builder.toString(); + } + + public static Map> getMap(final String queryParam) { + if (queryParam == null || queryParam.isBlank()) { + return Collections.emptyMap(); + } + + return Arrays.stream(StringUtils.split(queryParam, "&;")) + .map(QueryParameterParser::splitQueryParameter) + .collect(Collectors.groupingBy(SimpleImmutableEntry::getKey, HashMap::new, mapping(Map.Entry::getValue, toList()))); + } + + public static @NotNull MimeType getMimeType(final String queryString) { + final List mimeTypeList = QueryParameterParser.getMap(queryString).get(MIME_TYPE_TAG); + return mimeTypeList == null || mimeTypeList.isEmpty() ? MimeType.UNKNOWN : MimeType.getEnum(mimeTypeList.get(mimeTypeList.size() - 1)); + } + + /** + * Parse query parameter t. + * + * @param generic storage class type to be returned + * @param clazz storage class type + * @param queryString a rfc3986 query parameter string + * @return PoJo with those parameters that could be matched (N.B. flat map only) + * @throws NoSuchMethodException in case the class does not have a accessible constructor + * @throws IllegalAccessException in case the class cannot be instantiated + * @throws InvocationTargetException in case the class cannot be instantiated + * @throws InstantiationException in case the class cannot be instantiated + */ + public static T parseQueryParameter(Class clazz, final String queryString) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + final ClassFieldDescription fieldDescription = ClassUtils.getFieldDescription(clazz); + final Constructor constructor = clazz.getDeclaredConstructor(); + constructor.setAccessible(true); // NOSONAR NOPMD + final T obj = constructor.newInstance(); + final Map> queryMap = getMap(queryString); + for (FieldDescription f : fieldDescription.getChildren()) { + ClassFieldDescription field = (ClassFieldDescription) f; + final List values = queryMap.get(field.getFieldName()); + final TriConsumer mapFunction = STRING_TO_CLASS_CONVERTER.get(field.getType()); + if (mapFunction == null || values == null || values.isEmpty()) { + // skip field + continue; + } + final String value = values.get(values.size() - 1); + try { + mapFunction.accept(value, obj, field); + } catch (final Exception e) { // NOPMD exception is being rethrown + throw new IllegalArgumentException("error parsing value '" + value + "' for field: '" + clazz.getName() + "::" + field.getFieldName() + "'", e); + } + } + return obj; + } + + public static URI removeQueryParameter(URI oldUri, String removeQuery) throws URISyntaxException { + if (removeQuery == null || removeQuery.isBlank() || oldUri.getQuery() == null) { + return oldUri; + } + final Map> query = getMap(oldUri.getQuery()); + final int idx = removeQuery.indexOf('='); + if (idx >= 0) { + final String key = idx > 0 ? removeQuery.substring(0, idx) : removeQuery; + final String value = idx > 0 && removeQuery.length() > idx + 1 ? removeQuery.substring(idx + 1) : null; + final List entry = query.get(key); + if (entry != null) { + entry.remove(value); + if (entry.isEmpty()) { + query.remove(value); + } + } + } else { + query.remove(removeQuery); + } + final String newQueryParameter = QueryParameterParser.generateQueryParameter(query); + return new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(), newQueryParameter, oldUri.getFragment()); + } + + /** + * used as lambda expression for user-level code to read/write data into the query pojo + * + * @author rstein + */ + public interface TriConsumer { + /** + * Performs this operation on the given arguments. + * + * @param str the reference string + * @param rootObj the specific root object reference the given field is part of + * @param field the description for the given class member, if null then rootObj is written/read directly + */ + void accept(String str, Object rootObj, ClassFieldDescription field); + } + + @SuppressWarnings("PMD.DefaultPackage") + static SimpleImmutableEntry splitQueryParameter(String queryParameter) { // NOPMD package private for unit-testing purposes + final int idx = queryParameter.indexOf('='); + final String key = idx > 0 ? queryParameter.substring(0, idx) : queryParameter; + final String value = idx > 0 && queryParameter.length() > idx + 1 ? queryParameter.substring(idx + 1) : null; + return new SimpleImmutableEntry<>(URLDecoder.decode(key, UTF_8), value == null ? null : URLDecoder.decode(value, UTF_8)); + } +} diff --git a/core/src/main/java/io/opencmw/RingBufferEvent.java b/core/src/main/java/io/opencmw/RingBufferEvent.java new file mode 100644 index 00000000..f33b045f --- /dev/null +++ b/core/src/main/java/io/opencmw/RingBufferEvent.java @@ -0,0 +1,222 @@ +package io.opencmw; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.function.Predicate; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.utils.SharedPointer; + +import com.lmax.disruptor.EventHandler; + +@SuppressWarnings("PMD.TooManyMethods") +public class RingBufferEvent implements FilterPredicate, Cloneable { + private final static Logger LOGGER = LoggerFactory.getLogger(RingBufferEvent.class); + /** + * local UTC event arrival time-stamp [ms] + */ + public long arrivalTimeStamp; + + /** + * reference to the parent's disruptor sequence ID number + */ + public long parentSequenceNumber; + + /** + * list of known filters. N.B. this + */ + public final Filter[] filters; + + /** + * domain object carried by this ring buffer event + */ + public SharedPointer payload; + + /** + * collection of exceptions that have been issued while handling this RingBuffer event + */ + public final List throwables = new ArrayList<>(); + + /** + * + * @param filterConfig static filter configuration + */ + @SafeVarargs + public RingBufferEvent(final Class... filterConfig) { + assert filterConfig != null; + this.filters = new Filter[filterConfig.length]; + for (int i = 0; i < filters.length; i++) { + try { + filters[i] = filterConfig[i].getConstructor().newInstance(); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new IllegalArgumentException("filter initialisations error - could not instantiate class:" + filterConfig[i], e); + } + } + clear(); + } + + @Override + @SuppressWarnings({ "unchecked", "MethodDoesntCallSuperMethod" }) + public RingBufferEvent clone() { // NOSONAR NOPMD we do not want to call super (would be kind of stupid) + final RingBufferEvent retVal = new RingBufferEvent(Arrays.stream(filters).map(Filter::getClass).toArray(Class[] ::new)); + this.copyTo(retVal); + return retVal; + } + + public void copyTo(RingBufferEvent other) { + other.arrivalTimeStamp = arrivalTimeStamp; + other.parentSequenceNumber = parentSequenceNumber; + for (int i = 0; i < other.filters.length; i++) { + filters[i].copyTo(other.filters[i]); + } + other.payload = payload == null ? null : payload.getCopy(); + other.throwables.clear(); + other.throwables.addAll(throwables); + } + + public T getFilter(final Class filterType) { + for (Filter filter : filters) { + if (filter.getClass().isAssignableFrom(filterType)) { + return filterType.cast(filter); + } + } + final StringBuilder builder = new StringBuilder(); + builder.append("requested filter type '").append(filterType.getSimpleName()).append(" not part of ").append(RingBufferEvent.class.getSimpleName()).append(" definition: "); + printToStringArrayList(builder, "[", "]", (Object[]) filters); + throw new IllegalArgumentException(builder.toString()); + } + + public boolean matches(final Predicate predicate) { + return predicate.test(this); + } + + public boolean matches(Class filterType, final Predicate predicate) { + return predicate.test(getFilter(filterType)); + } + + /** + * @param payloadType required payload class-type + * @return {@code true} if payload is defined and matches type + */ + public boolean matches(Class payloadType) { + return payload != null && payload.getType() != null && payloadType.isAssignableFrom(payload.getType()); + } + + public final void clear() { + arrivalTimeStamp = 0L; + parentSequenceNumber = -1L; + for (Filter filter : filters) { + filter.clear(); + } + throwables.clear(); + if (payload != null) { + payload.release(); + } + payload = null; // NOPMD - null use on purpose (faster/easier than an Optional) + } + + @Override + public boolean test(final Class filterClass, final Predicate filterPredicate) { + return filterPredicate.test(filterClass.cast(getFilter(filterClass))); + } + + @Override + public String toString() { + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.UK); + final StringBuilder builder = new StringBuilder(); + builder.append(RingBufferEvent.class.getSimpleName()).append(": arrivalTimeStamp ").append(arrivalTimeStamp).append(" (").append(sdf.format(arrivalTimeStamp)).append(") parent sequence number: ").append(parentSequenceNumber).append(" - filter: "); + printToStringArrayList(builder, "[", "]", (Object[]) filters); + if (!throwables.isEmpty()) { + builder.append(" - exceptions (n=").append(throwables.size()).append("):\n"); + for (Throwable t : throwables) { + builder.append(getPrintableStackTrace(t)).append('\n'); + } + } + return builder.toString(); + } + + public static void printToStringArrayList(final StringBuilder builder, final String prefix, final String postFix, final Object... items) { + if (prefix != null && !prefix.isBlank()) { + builder.append(prefix); + } + boolean more = false; + for (Object o : items) { + if (more) { + builder.append(", "); + } + builder.append(o.getClass().getSimpleName()).append(':').append(o.toString()); + more = true; + } + if (postFix != null && !postFix.isBlank()) { + builder.append(postFix); + } + //TODO: refactor into a common utility class + } + + public static String getPrintableStackTrace(final Throwable t) { + if (t == null) { + return ""; + } + final StringWriter sw = new StringWriter(); + t.printStackTrace(new PrintWriter(sw)); + return sw.toString(); + //TODO: refactor into a common utility class + } + + /** + * default buffer element clearing handler + */ + public static class ClearEventHandler implements EventHandler { + @Override + public void onEvent(RingBufferEvent event, long sequence, boolean endOfBatch) { + LOGGER.atTrace().addArgument(sequence).addArgument(endOfBatch).log("clearing RingBufferEvent sequence = {} endOfBatch = {}"); + event.clear(); + } + } + + @Override + public boolean equals(final Object obj) { // NOSONAR NOPMD - npath complexity unavoidable for performance reasons + if (this == obj) { + return true; + } + if (!(obj instanceof RingBufferEvent)) { + return false; + } + final RingBufferEvent other = (RingBufferEvent) obj; + if (hashCode() != other.hashCode()) { + return false; + } + if (arrivalTimeStamp != other.arrivalTimeStamp) { + return false; + } + if (parentSequenceNumber != other.parentSequenceNumber) { + return false; + } + if (!Arrays.equals(filters, other.filters)) { + return false; + } + if (!Objects.equals(payload, other.payload)) { + return false; + } + return throwables.equals(other.throwables); + } + + @Override + public int hashCode() { + int result = (int) (arrivalTimeStamp ^ (arrivalTimeStamp >>> 32)); + result = 31 * result + (int) (parentSequenceNumber ^ (parentSequenceNumber >>> 32)); + result = 31 * result + Arrays.hashCode(filters); + result = 31 * result + (payload == null ? 0 : payload.hashCode()); + result = 31 * result + throwables.hashCode(); + return result; + } +} diff --git a/core/src/main/java/io/opencmw/domain/BinaryData.java b/core/src/main/java/io/opencmw/domain/BinaryData.java new file mode 100644 index 00000000..9ebb7372 --- /dev/null +++ b/core/src/main/java/io/opencmw/domain/BinaryData.java @@ -0,0 +1,130 @@ +package io.opencmw.domain; + +import java.util.Arrays; +import java.util.Objects; + +import org.zeromq.util.ZData; + +import io.opencmw.MimeType; +import io.opencmw.serialiser.annotations.MetaInfo; + +/** + * basic domain object definition for receiving or sending generic binary data + */ +@MetaInfo(description = "domain object definition for receiving/sending generic binary data") +public class BinaryData { + public String resourceName = "default"; + public MimeType contentType = MimeType.BINARY; + public byte[] data = {}; + public int dataSize = -1; + + public BinaryData() { + // default constructor + } + + public BinaryData(final String resourceName, final MimeType contentType, final byte[] data) { + this(resourceName, contentType, data, -1); + } + + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + public BinaryData(final String resourceName, final MimeType contentType, final byte[] data, final int dataSize) { + this.resourceName = resourceName; + this.contentType = contentType; + this.data = data; + this.dataSize = dataSize; + checkConsistency(); // NOPMD + } + + public void checkConsistency() { + if (resourceName == null || resourceName.isBlank()) { + throw new IllegalArgumentException("resourceName must not be blank"); + } + if (contentType == null) { + throw new IllegalArgumentException("mimeType must not be blank"); + } + if (data == null || (dataSize >= 0 && data.length < dataSize)) { + throw new IllegalArgumentException("data[" + (data == null ? "null" : data.length) + "] must be larger than dataSize=" + dataSize); + } + } + + @Override + public String toString() { + return "BinaryData{resourceName='" + resourceName + "', contentType=" + contentType.name() + "(\"" + contentType + "\"), dataSize=" + dataSize + ", data=" + ZData.toString(data) + '}'; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BinaryData)) { + return false; + } + final BinaryData that = (BinaryData) o; + + if (!Objects.equals(resourceName, that.resourceName) || contentType != that.contentType + || (data == null && that.data != null) || (data != null && that.data == null)) { + return false; + } + if (data == null) { + return true; + } + final int minSize = dataSize >= 0 ? Math.min(data.length, dataSize) : data.length; + return Arrays.equals(data, 0, minSize, that.data, 0, that.data.length); + } + + @Override + public int hashCode() { + int result = resourceName == null ? 0 : resourceName.hashCode(); + result = 31 * result + (contentType == null ? 0 : contentType.hashCode()); + result = 31 * result + Arrays.hashCode(data); + return result; + } + + protected static String fixPreAndPost(final String name) { + final String nonNullName = name == null ? "/" : name.trim(); + final String fixedPrefix = (nonNullName.startsWith("/") ? nonNullName : '/' + nonNullName); + return fixedPrefix.endsWith("/") ? fixedPrefix : fixedPrefix + '/'; + } + + protected static String genExportName(final String name) { + checkField("genExportName(name)", name); + int p = name.lastIndexOf('/'); + if (p < 0) { + p = 0; + } + int e = name.lastIndexOf('.'); + if (e < 0) { + e = name.length(); + } + return name.substring(p, e).replace("/", ""); + } + + protected static String genExportNameData(final String name) { + checkField("genExportNameData(name)", name); + int p = name.lastIndexOf('/'); + if (p < 0) { + p = 0; + } + return name.substring(p).replace("/", ""); + } + + protected static String getCategory(final String name) { + checkField("getCategory(name)", name); + final int p = name.lastIndexOf('/'); + if (p < 0) { + return fixPreAndPost(""); + } + return fixPreAndPost(name.substring(0, p + 1)); + } + + private static String checkField(final String field, final String category) { + if (category == null) { + throw new IllegalArgumentException(field + "category not be null"); + } + if (category.isBlank()) { + throw new IllegalArgumentException(field + "must not be blank"); + } + return category; + } +} diff --git a/core/src/main/java/io/opencmw/domain/NoData.java b/core/src/main/java/io/opencmw/domain/NoData.java new file mode 100644 index 00000000..2d8e7af0 --- /dev/null +++ b/core/src/main/java/io/opencmw/domain/NoData.java @@ -0,0 +1,10 @@ +package io.opencmw.domain; + +import io.opencmw.serialiser.annotations.MetaInfo; + +/** + * dummy domain object definition to indicate that no input/output data is requested + */ +@MetaInfo(description = "dummy domain object definition to indicate that no input/output data is requested") +public class NoData { +} diff --git a/core/src/main/java/io/opencmw/filter/EvtTypeFilter.java b/core/src/main/java/io/opencmw/filter/EvtTypeFilter.java new file mode 100644 index 00000000..c0d3f57f --- /dev/null +++ b/core/src/main/java/io/opencmw/filter/EvtTypeFilter.java @@ -0,0 +1,96 @@ +package io.opencmw.filter; + +import java.util.Objects; +import java.util.function.Predicate; + +import io.opencmw.Filter; + +public class EvtTypeFilter implements Filter { + public DataType evtType = DataType.UNKNOWN; + public UpdateType updateType = UpdateType.UNKNOWN; + public String typeName = ""; + protected int hashCode = 0; // NOPMD + + @Override + public void clear() { + hashCode = 0; + evtType = DataType.UNKNOWN; + updateType = UpdateType.UNKNOWN; + typeName = ""; + } + + @Override + public void copyTo(final Filter other) { + if (!(other instanceof EvtTypeFilter)) { + return; + } + ((EvtTypeFilter) other).hashCode = this.hashCode; + ((EvtTypeFilter) other).evtType = this.evtType; + ((EvtTypeFilter) other).typeName = this.typeName; + ((EvtTypeFilter) other).updateType = this.updateType; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof EvtTypeFilter)) { + return false; + } + final EvtTypeFilter other = (EvtTypeFilter) obj; + return evtType == other.evtType && updateType == other.updateType && Objects.equals(typeName, other.typeName); + } + + @Override + public int hashCode() { + return hashCode == 0 ? hashCode = Objects.hash(evtType, updateType, typeName) : hashCode; + } + + @Override + public String toString() { + return '[' + EvtTypeFilter.class.getSimpleName() + ": evtType=" + evtType + " typeName='" + typeName + "']"; + } + + public enum DataType { + TIMING_EVENT, + AGGREGATE_DATA, + DEVICE_DATA, + SETTING_SUPPLY_DATA, + PROCESSED_DATA, + OTHER, + UNKNOWN + } + + public enum UpdateType { + EMPTY, + PARTIAL, + COMPLETE, + OTHER, + UNKNOWN + } + + public static Predicate isTimingData() { + return t -> t.evtType == DataType.TIMING_EVENT; + } + + public static Predicate isTimingData(final String typeName) { + return t -> t.evtType == DataType.TIMING_EVENT && Objects.equals(t.typeName, typeName); + } + + public static Predicate isDeviceData() { + return t -> t.evtType == DataType.DEVICE_DATA; + } + + public static Predicate isDeviceData(final String typeName) { + return t -> t.evtType == DataType.DEVICE_DATA && Objects.equals(t.typeName, typeName); + } + + public static Predicate isSettingsData() { + return t -> t.evtType == DataType.SETTING_SUPPLY_DATA; + } + + public static Predicate isSettingsData(final String typeName) { + return t -> t.evtType == DataType.SETTING_SUPPLY_DATA && Objects.equals(t.typeName, typeName); + } +} diff --git a/core/src/main/java/io/opencmw/filter/TimingCtx.java b/core/src/main/java/io/opencmw/filter/TimingCtx.java new file mode 100644 index 00000000..47d824a0 --- /dev/null +++ b/core/src/main/java/io/opencmw/filter/TimingCtx.java @@ -0,0 +1,206 @@ +package io.opencmw.filter; + +import java.text.SimpleDateFormat; +import java.util.Locale; +import java.util.Objects; +import java.util.function.Predicate; + +import org.apache.commons.lang3.StringUtils; + +import io.opencmw.Filter; + +import com.jsoniter.spi.JsoniterSpi; + +@SuppressWarnings({ "PMD.TooManyMethods" }) // - the nature of this class definition +public class TimingCtx implements Filter { + public static final String WILD_CARD = "ALL"; + public static final int WILD_CARD_VALUE = -1; + public static final String SELECTOR_PREFIX = "FAIR.SELECTOR."; + /** selector string, e.g.: 'FAIR.SELECTOR.C=0:S=1:P=3:T=101' */ + public String selector = ""; + /** Beam-Production-Chain (BPC) ID - uninitialised/wildcard value = -1 */ + public int cid; + /** Sequence ID -- N.B. this is the timing sequence number not the disruptor sequence ID */ + public int sid; + /** Beam-Process ID (PID) - uninitialised/wildcard value = -1 */ + public int pid; + /** timing group ID - uninitialised/wildcard value = -1 */ + public int gid; + /** Beam-Production-Chain-Time-Stamp - UTC in [us] since 1.1.1980 */ + public long bpcts; + /** stores the settings-supply related ctx name */ + public String ctxName; + protected int hashCode = 0; // NOPMD cached hash code + static { + // custom JsonIter decoder + JsoniterSpi.registerTypeDecoder(TimingCtx.class, iter -> TimingCtx.get(iter.readString())); + } + public TimingCtx() { + clear(); // NOPMD -- called during initialisation + } + + @Override + public void clear() { + hashCode = 0; + selector = ""; + cid = -1; + sid = -1; + pid = -1; + gid = -1; + bpcts = -1; + ctxName = ""; + } + + @Override + public void copyTo(final Filter other) { + if (!(other instanceof TimingCtx)) { + return; + } + final TimingCtx otherCtx = (TimingCtx) other; + otherCtx.selector = this.selector; + otherCtx.cid = this.cid; + otherCtx.sid = this.sid; + otherCtx.pid = this.pid; + otherCtx.gid = this.gid; + otherCtx.bpcts = this.bpcts; + otherCtx.ctxName = this.ctxName; + otherCtx.hashCode = this.hashCode; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + final TimingCtx otherCtx = (TimingCtx) o; + if (hashCode != otherCtx.hashCode() || cid != otherCtx.cid || sid != otherCtx.sid || pid != otherCtx.pid || gid != otherCtx.gid || bpcts != otherCtx.bpcts) { + return false; + } + + return Objects.equals(selector, otherCtx.selector); + } + + @Override + public int hashCode() { + if (hashCode != 0) { + return hashCode; + } + hashCode = selector == null ? 0 : selector.hashCode(); + hashCode = 31 * hashCode + cid; + hashCode = 31 * hashCode + sid; + hashCode = 31 * hashCode + pid; + hashCode = 31 * hashCode + gid; + hashCode = 31 * hashCode + Long.hashCode(bpcts); + return hashCode; + } + + public Predicate matches(final TimingCtx other) { + return t -> this.equals(other); + } + + /** + * TODO: add more thorough documentation or reference + * + * @param selector new selector to be parsed, e.g. 'FAIR.SELECTOR.ALL', 'FAIR.SELECTOR.C=1:S=3:P:3:T:103' + * @param bpcts beam-production-chain time-stamp [us] + */ + @SuppressWarnings("PMD.NPathComplexity") // -- parser/format has intrinsically large number of possible combinations + public void setSelector(final String selector, final long bpcts) { + if (bpcts < 0) { + throw new IllegalArgumentException("BPCTS time stamp < 0 :" + bpcts); + } + try { + clear(); + this.selector = Objects.requireNonNull(selector, "selector string must not be null"); + this.bpcts = bpcts; + + final String selectorUpper = selector.toUpperCase(Locale.UK); + if (selector.isBlank() || WILD_CARD.equals(selectorUpper)) { + return; + } + + final String[] identifiers = StringUtils.replace(selectorUpper, SELECTOR_PREFIX, "", 1).split(":"); + if (identifiers.length == 1 && WILD_CARD.equals(identifiers[0])) { + return; + } + + for (String tag : identifiers) { + final String[] splitSubComponent = tag.split("="); + assert splitSubComponent.length == 2 : "invalid selector: " + selector; // NOPMD NOSONAR assert only while debugging + final int value = splitSubComponent[1].equals(WILD_CARD) ? -1 : Integer.parseInt(splitSubComponent[1]); + switch (splitSubComponent[0]) { + case "C": + this.cid = value; + break; + case "S": + this.sid = value; + break; + case "P": + this.pid = value; + break; + case "T": + this.gid = value; + break; + default: + clear(); + throw new IllegalArgumentException("cannot parse selector: '" + selector + "' sub-tag: " + tag); + } + } + } catch (Throwable t) { // NOPMD NOSONAR should catch Throwable + clear(); + throw new IllegalArgumentException("Invalid selector or bpcts: " + selector, t); + } + } + + public static TimingCtx get(final String ctxString) { + final TimingCtx ctx = new TimingCtx(); + ctx.setSelector(ctxString, 0); + return ctx; + } + + @Override + public String toString() { + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.UK); + return '[' + TimingCtx.class.getSimpleName() + ": bpcts=" + bpcts + " (\"" + sdf.format(bpcts / 1_000_000) + "\"), selector='" + selector + "', cid=" + cid + ", sid=" + sid + ", pid=" + pid + ", gid=" + gid + ']'; + } + + public static Predicate matches(final int cid, final int sid, final int pid, final long bpcts) { + return t -> t.bpcts == bpcts && t.cid == cid && wildCardMatch(t.sid, sid) && wildCardMatch(t.pid, pid); + } + + public static Predicate matches(final int cid, final int sid, final long bpcts) { + return t -> t.bpcts == bpcts && wildCardMatch(t.cid, cid) && wildCardMatch(t.sid, sid); + } + + public static Predicate matches(final int cid, final long bpcts) { + return t -> t.bpcts == bpcts && wildCardMatch(t.cid, cid); + } + + public static Predicate matches(final int cid, final int sid, final int pid) { + return t -> wildCardMatch(t.cid, cid) && wildCardMatch(t.sid, sid) && wildCardMatch(t.pid, pid); + } + + public static Predicate matches(final int cid, final int sid) { + return t -> wildCardMatch(t.cid, cid) && wildCardMatch(t.sid, sid); + } + + public static Predicate matchesBpcts(final long bpcts) { + return t -> t.bpcts == bpcts; + } + + public static Predicate isOlderBpcts(final long bpcts) { + return t -> t.bpcts < bpcts; + } + + public static Predicate isNewerBpcts(final long bpcts) { + return t -> t.bpcts > bpcts; + } + + protected static boolean wildCardMatch(final int a, final int b) { + return a == b || a == WILD_CARD_VALUE || b == WILD_CARD_VALUE; + } +} diff --git a/core/src/main/java/io/opencmw/rbac/BasicRbacRole.java b/core/src/main/java/io/opencmw/rbac/BasicRbacRole.java new file mode 100644 index 00000000..0235575b --- /dev/null +++ b/core/src/main/java/io/opencmw/rbac/BasicRbacRole.java @@ -0,0 +1,45 @@ +package io.opencmw.rbac; + +import java.util.Locale; + +/** + * basic definition of common Role-Based-Access-Control (RBAC) roles + * + * original RBAC concept: + *
    + *
  • Ferraiolo, D.F. & Kuhn, D.R. (October 1992). "Role-Based Access Control". 15th National Computer Security Conference: 554–563. + * https://csrc.nist.gov/CSRC/media/Publications/conference-paper/1992/10/13/role-based-access-controls/documents/ferraiolo-kuhn-92.pdf + *
    • + *
  • Sandhu, R., Coyne, E.J., Feinstein, H.L. and Youman, C.E. (August 1996). "Role-Based Access Control Models". IEEE Computer. 29 (2): 38–47. CiteSeerX 10.1.1.50.7649. doi:10.1109/2.485845 + * https://csrc.nist.gov/projects/role-based-access-control + *
    • + *
+ */ +public enum BasicRbacRole implements RbacRole { + ADMIN(0), // highest priority in queues + READ_WRITE(100), + READ_ONLY(200), + ANYONE(300), + NULL(300); // lowest priority in queues + + private final int priority; + + BasicRbacRole(final int priority) { + this.priority = priority; + } + + @Override + public String getName() { + return this.toString(); + } + + @Override + public int getPriority() { + return this.priority; + } + + @Override + public BasicRbacRole getRole(final String roleName) { + return valueOf(roleName.toUpperCase(Locale.UK)); + } +} \ No newline at end of file diff --git a/core/src/main/java/io/opencmw/rbac/RbacProvider.java b/core/src/main/java/io/opencmw/rbac/RbacProvider.java new file mode 100644 index 00000000..a193644b --- /dev/null +++ b/core/src/main/java/io/opencmw/rbac/RbacProvider.java @@ -0,0 +1,7 @@ +package io.opencmw.rbac; + +/** + * Interface for RBAC implementations to sign messages with public key cryptography + */ +public interface RbacProvider { +} diff --git a/core/src/main/java/io/opencmw/rbac/RbacRole.java b/core/src/main/java/io/opencmw/rbac/RbacRole.java new file mode 100644 index 00000000..7288c5a4 --- /dev/null +++ b/core/src/main/java/io/opencmw/rbac/RbacRole.java @@ -0,0 +1,70 @@ +package io.opencmw.rbac; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Collectors; + +import org.jetbrains.annotations.NotNull; + +/** + * Interface for Role-Based-Access-Control (RBAC) roles + * + * original RBAC concept: + *
    + *
  • Ferraiolo, D.F. & Kuhn, D.R. (October 1992). "Role-Based Access Control". 15th National Computer Security Conference: 554–563. + * https://csrc.nist.gov/CSRC/media/Publications/conference-paper/1992/10/13/role-based-access-controls/documents/ferraiolo-kuhn-92.pdf + *
    • + *
  • Sandhu, R., Coyne, E.J., Feinstein, H.L. and Youman, C.E. (August 1996). "Role-Based Access Control Models". IEEE Computer. 29 (2): 38–47. CiteSeerX 10.1.1.50.7649. doi:10.1109/2.485845 + * https://csrc.nist.gov/projects/role-based-access-control + *
    • + *
+ */ +public interface RbacRole> extends Comparable { + default String getRoles(final Set roleSet) { + return roleSet.stream().map(RbacRole::toString).collect(Collectors.joining(", ")); + } + + default Set getRoles(final String roleString) { + if (roleString.contains(":")) { + throw new IllegalArgumentException("roleString must not contain [:]"); + } + + final HashSet roles = new HashSet<>(); + for (final String role : roleString.replaceAll("\\s", "").split(",")) { + if (role == null || role.isEmpty() || "*".equals(role)) { // NOPMD + continue; + } + roles.add(getRole(role.toUpperCase(Locale.UK))); + } + + return Collections.unmodifiableSet(roles); + } + + T getRole(String roleName); + + /** + * + * @return role name + */ + String getName(); + + /** + * + * @return role priority used to schedule tasks or position in queues ( smaller numbers == higher importance) + */ + int getPriority(); + + @Override + default int compareTo(@NotNull RbacRole otherRole) { + System.err.println("T = " + otherRole); + if (getPriority() > otherRole.getPriority()) { + return 1; + } + if (getPriority() == otherRole.getPriority()) { + return 0; + } + return 1; + } +} diff --git a/core/src/main/java/io/opencmw/rbac/RbacToken.java b/core/src/main/java/io/opencmw/rbac/RbacToken.java new file mode 100644 index 00000000..ebc85801 --- /dev/null +++ b/core/src/main/java/io/opencmw/rbac/RbacToken.java @@ -0,0 +1,66 @@ +package io.opencmw.rbac; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.zeromq.ZMQ; + +public class RbacToken { + private final static String RBAC_TOKEN_PREFIX = "RBAC"; + private final String signedHashCode; + private final RbacRole> rbacRole; + private final String stringRepresentation; + private final byte[] byteRepresentation; + + public RbacToken(final RbacRole> rbacRole, final String signedHashCode) { + if (rbacRole == null) { + throw new IllegalArgumentException("rbacRole must not be null: " + null); + } + if (signedHashCode == null) { + throw new IllegalArgumentException("signedHashCode must not be null: " + null); + } + this.rbacRole = rbacRole; + this.signedHashCode = signedHashCode; + this.stringRepresentation = RBAC_TOKEN_PREFIX + "=" + this.rbacRole.getName() + "," + signedHashCode; + this.byteRepresentation = stringRepresentation.getBytes(StandardCharsets.UTF_8); + + // BCrypt.hashpw() + } + + public RbacRole> getRole() { + return rbacRole; + } + + public String getSignedHashCode() { + return signedHashCode; + } + + @Override + public String toString() { + return stringRepresentation; + } + + public byte[] getBytes() { + return Arrays.copyOf(byteRepresentation, byteRepresentation.length); + } + + public static RbacToken from(final byte[] rbacToken) { + return from(rbacToken, rbacToken.length); + } + + public static RbacToken from(final byte[] rbacToken, final int length) { + return from(new String(rbacToken, 0, length, ZMQ.CHARSET)); + } + + public static RbacToken from(final String rbacToken) { + if (rbacToken == null || rbacToken.isBlank()) { + return new RbacToken(BasicRbacRole.ANYONE, ""); + } + final String[] component = rbacToken.split("[,=]"); + if (component.length != 3 || !RBAC_TOKEN_PREFIX.equals(component[0])) { + // protocol error: sent token with less or more than two commas + return new RbacToken(BasicRbacRole.NULL, ""); + } + return new RbacToken(BasicRbacRole.NULL.getRole(component[1]), component[2]); + } +} diff --git a/core/src/main/java/io/opencmw/utils/AnsiDefs.java b/core/src/main/java/io/opencmw/utils/AnsiDefs.java new file mode 100644 index 00000000..e71b4897 --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/AnsiDefs.java @@ -0,0 +1,17 @@ +package io.opencmw.utils; + +/** + * basic ANSI colour code and other escape character definitions @see https://en.wikipedia.org/wiki/ANSI_escape_code + */ +public class AnsiDefs { // NOPMD - nomen est omen + public static final int MIN_PRINTABLE_CHAR = 32; + public static final String ANSI_RESET = "\u001B[0m"; + public static final String ANSI_BLACK = "\u001B[30m"; + public static final String ANSI_RED = "\u001B[31m"; + public static final String ANSI_GREEN = "\u001B[32m"; + public static final String ANSI_YELLOW = "\u001B[33m"; + public static final String ANSI_BLUE = "\u001B[34m"; + public static final String ANSI_PURPLE = "\u001B[35m"; + public static final String ANSI_CYAN = "\u001B[36m"; + public static final String ANSI_WHITE = "\u001B[37m"; +} diff --git a/core/src/main/java/io/opencmw/utils/Cache.java b/core/src/main/java/io/opencmw/utils/Cache.java new file mode 100644 index 00000000..beea949d --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/Cache.java @@ -0,0 +1,374 @@ +package io.opencmw.utils; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +import org.jetbrains.annotations.NotNull; + +/** + * A simple map based cache with timeOut and limit + * + * usage example: + * + * 
+ *  {@code
+ *     public class Demo {
+ *         private Cache cache;
+ *
+ *         public Demo() {
+ *             cache = final Cache cache = Cache.builder().withLimit(10)
+ *                  .withTimeout(100, TimeUnit.MILLISECONDS).build();
+ *             // alternatively:
+ *             // cache = new Cache(100, TimeUnit.MILLISECONDS, 10);
+ *
+ *             String name1 = "Han Solo";
+ *
+ *             cache.put(name1, 10);
+ *
+ *             System.out.println(name1 + " is cached: " + isCached(name1));
+ *
+ *             // Wait 1 second
+ *             try {
+ *                 Thread.sleep(1000);
+ *             } catch (InterruptedException e) {
+ *                 e.printStackTrace();
+ *             }
+ *
+ *             System.out.println(name1 + " is cached: " + isCached(name1));
+ *
+ *             // Wait another second
+ *             try {
+ *                 Thread.sleep(1000);
+ *             } catch (InterruptedException e) {
+ *                 e.printStackTrace();
+ *             }
+ *
+ *             System.out.println(name1 + " is cached: " + isCached(name1));
+ *         }
+ *
+ *         private boolean isCached(final String KEY) {
+ *             return cache.get(KEY).isPresent();
+ *         }
+ *
+ *         public static void main(String[] args) {
+ *             new Demo();
+ *         }
+ *     }
+ * }
+ *
+ * + * + * Original code courtesy from: https://github.com/HanSolo/cache + * + * @author Gerrit Grunwald (aka. HanSolo, original concept) + * @author rstein + * + * @param search key + * @param cached value + */ +@SuppressWarnings({ "PMD.DoNotUseThreads", "PMD.TooManyMethods" }) // thread use necessary for maintenance tasks, methods due to Map interface +public class Cache implements Map { + private final ConcurrentHashMap dataCache; + private final ConcurrentHashMap timeOutMap; + private final ChronoUnit chronoUnit; + private final TimeUnit timeUnit; + private final long timeOut; + private final int limit; + private final BiConsumer preListener; + private final BiConsumer postListener; + + public Cache(final int limit) { + this(0, TimeUnit.MILLISECONDS, limit, null, null); + } + + public Cache(final long timeOut, final TimeUnit timeUnit) { + this(timeOut, timeUnit, Integer.MAX_VALUE, null, null); + } + + public Cache(final long timeOut, final TimeUnit timeUnit, final int limit) { + this(timeOut, timeUnit, limit, null, null); + } + + private Cache(final long timeOut, final TimeUnit timeUnit, final int limit, final BiConsumer preListener, final BiConsumer postListener) { + dataCache = new ConcurrentHashMap<>(); + timeOutMap = new ConcurrentHashMap<>(); + + if (timeOut < 0) { + throw new IllegalArgumentException("Timeout cannot be negative"); + } + if (timeOut > 0 && null == timeUnit) { + throw new IllegalArgumentException("TimeUnit cannot be null if timeOut is > 0"); + } + if (limit < 1) { + throw new IllegalArgumentException("Limit cannot be smaller than 1"); + } + + this.timeOut = timeOut; + this.timeUnit = timeUnit; + chronoUnit = convertToChronoUnit(timeUnit); + this.limit = limit; + + this.preListener = preListener; + this.postListener = postListener; + + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> { + final Thread t = Executors.defaultThreadFactory().newThread(r); + t.setName(Cache.class.getCanonicalName() + "-Thread"); + t.setDaemon(true); + return t; + }); // Daemon Service + + if (timeOut != 0) { + executor.scheduleAtFixedRate(this::checkTime, 0, timeOut, timeUnit); + } + } + + @Override + public void clear() { + dataCache.clear(); + timeOutMap.clear(); + } + + @Override + public boolean containsKey(final Object key) { + return dataCache.containsKey(key); + } + + @Override + public boolean containsValue(final Object value) { + return dataCache.containsValue(value); + } + + @Override + public @NotNull Set> entrySet() { + return dataCache.entrySet(); + } + + @Override + @SuppressWarnings("unchecked") + public V get(final Object key) { + return getIfPresent((K) key); + } + + public V getIfPresent(final K key) { + timeOutMap.put(key, Instant.now()); + return dataCache.getOrDefault(key, null); + } + + public long getLimit() { + return limit; + } + + public Optional getOptional(final K key) { + return Optional.ofNullable(getIfPresent(key)); + } + + public int getSize() { + return dataCache.size(); + } + + public long getTimeout() { + return timeOut; + } + + public TimeUnit getTimeUnit() { + return timeUnit; + } + + @Override + public boolean isEmpty() { + return dataCache.isEmpty(); + } + + @Override + public @NotNull Set keySet() { + return dataCache.keySet(); + } + + @Override + public V put(final K key, final V value) { + checkSize(); + final V val = dataCache.put(key, value); + timeOutMap.put(key, Instant.now()); + return val; + } + + @Override + public V putIfAbsent(final K key, final V value) { + checkSize(); + final V val = dataCache.putIfAbsent(key, value); + timeOutMap.putIfAbsent(key, Instant.now()); + return val; + } + + @Override + public void putAll(final Map m) { + checkSize(m.size()); + dataCache.putAll(m); + final Instant now = Instant.now(); + m.keySet().forEach(key -> timeOutMap.putIfAbsent(key, now)); + } + + @Override + public V remove(final Object key) { + final V val = dataCache.remove(key); + timeOutMap.remove(key); + return val; + } + + @Override + public int size() { + return dataCache.size(); + } + + @Override + public @NotNull Collection values() { + return dataCache.values(); + } + + protected void checkSize() { + checkSize(1); + } + + protected void checkSize(final int nNewElements) { + if (dataCache.size() < limit) { + return; + } + final int surplusEntries = Math.max(dataCache.size() - limit + nNewElements, 0); + final List toBeRemoved = timeOutMap.entrySet().stream().sorted(Entry.comparingByValue().reversed()).limit(surplusEntries).map(Entry::getKey).collect(Collectors.toList()); + removeEntries(toBeRemoved); + } + + protected void checkTime() { + final Instant cutoffTime = Instant.now().minus(timeOut, chronoUnit); + final List toBeRemoved = timeOutMap.entrySet().stream().filter(entry -> entry.getValue().isBefore(cutoffTime)).map(Entry::getKey).collect(Collectors.toList()); + removeEntries(toBeRemoved); + } + + private void removeEntries(final List toBeRemoved) { + final HashMap removalMap; + if (preListener == null && postListener == null) { + removalMap = null; + } else { + removalMap = new HashMap<>(); + toBeRemoved.forEach(key -> removalMap.put(key, dataCache.get(key))); + } + + // call registered pre-listener + if (preListener != null) { + removalMap.forEach(preListener); + } + + toBeRemoved.forEach(key -> { + timeOutMap.remove(key); + dataCache.remove(key); + }); + + // call registered post-listener + if (postListener != null) { + removalMap.forEach(postListener); + } + } + + public static CacheBuilder builder() { + return new CacheBuilder<>(); + } + + protected static int clamp(final int min, final int max, final int value) { + if (value < min) { + return min; + } + return Math.min(value, max); + } + + protected static long clamp(final long min, final long max, final long value) { + if (value < min) { + return min; + } + return Math.min(value, max); + } + + protected static ChronoUnit convertToChronoUnit(final TimeUnit timeUnit) { + switch (timeUnit) { + case NANOSECONDS: + return ChronoUnit.NANOS; + case MICROSECONDS: + return ChronoUnit.MICROS; + case SECONDS: + return ChronoUnit.SECONDS; + case MINUTES: + return ChronoUnit.MINUTES; + case HOURS: + return ChronoUnit.HOURS; + case DAYS: + return ChronoUnit.DAYS; + case MILLISECONDS: + default: + return ChronoUnit.MILLIS; + } + } + + public static class CacheBuilder { + private int limit = Integer.MAX_VALUE; + private long timeOut; + private TimeUnit timeUnit = TimeUnit.MILLISECONDS; + private BiConsumer preListener; + private BiConsumer postListener; + + private CacheBuilder() { + // only called via builderCacheRemovalListener + } + + public Cache build() { + return new Cache<>(timeOut, timeUnit, limit, preListener, postListener); + } + + public CacheBuilder withLimit(final int limit) { + if (limit < 1) { + throw new IllegalArgumentException("Limit cannot be smaller than 1"); + } + this.limit = limit; + return this; + } + + public CacheBuilder withPostListener(final BiConsumer listener) { + if (listener == null) { + throw new IllegalArgumentException("listener cannot be null"); + } + this.postListener = listener; + return this; + } + + public CacheBuilder withPreListener(final BiConsumer listener) { + if (listener == null) { + throw new IllegalArgumentException("listener cannot be null"); + } + this.preListener = listener; + return this; + } + + public CacheBuilder withTimeout(final long timeOut, final TimeUnit timeUnit) { + if (timeOut < 0) { + throw new IllegalArgumentException("Timeout cannot be negative"); + } + if (null == timeUnit) { + throw new IllegalArgumentException("TimeUnit cannot be null"); + } + this.timeOut = clamp(0, Integer.MAX_VALUE, timeOut); + this.timeUnit = timeUnit; + return this; + } + } +} diff --git a/core/src/main/java/io/opencmw/utils/CustomFuture.java b/core/src/main/java/io/opencmw/utils/CustomFuture.java new file mode 100644 index 00000000..a57ebb40 --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/CustomFuture.java @@ -0,0 +1,118 @@ +package io.opencmw.utils; + +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +public class CustomFuture implements Future { + private static final String FUTURE_HAS_BEEN_CANCELLED = "future has been cancelled"; + protected final Lock lock = new ReentrantLock(); + protected final Condition processorNotifyCondition = lock.newCondition(); + protected final AtomicBoolean done = new AtomicBoolean(false); + private final AtomicBoolean requestCancel = new AtomicBoolean(false); + private final AtomicBoolean cancelled = new AtomicBoolean(false); + private T reply; + private Throwable exception; + + @Override + public boolean cancel(final boolean mayInterruptIfRunning) { + if (done.getAndSet(true)) { + return false; + } + cancelled.set(true); + notifyListener(); + return !requestCancel.getAndSet(true); + } + + @Override + public T get() throws ExecutionException, InterruptedException { + try { + return get(0, TimeUnit.NANOSECONDS); + } catch (TimeoutException e) { + // cannot normally occur -- need this because we re-use 'get(...)' to avoid code duplication + throw new ExecutionException("TimeoutException should not occur here", e); + } + } + + @SuppressWarnings("NullableProblems") + @Override + public T get(final long timeout, final TimeUnit unit) throws ExecutionException, InterruptedException, TimeoutException { + if (cancelled.get()) { + throw new CancellationException(FUTURE_HAS_BEEN_CANCELLED); + } + + if (isDone()) { + if (exception == null) { + return reply; + } + throw new ExecutionException(exception); + } + lock.lock(); + try { + while (!isDone()) { + if (timeout > 0) { + if (!processorNotifyCondition.await(timeout, unit)) { + throw new TimeoutException(); + } + } else { + processorNotifyCondition.await(); + } + if (cancelled.get()) { + throw new CancellationException(FUTURE_HAS_BEEN_CANCELLED); + } + } + } finally { + lock.unlock(); + } + if (exception != null) { + throw new ExecutionException(exception); + } + return reply; + } + + @Override + public boolean isCancelled() { + return cancelled.get(); + } + + @Override + public boolean isDone() { + return done.get(); + } + + /** + * set reply and notify potential listeners + * @param newValue the new value to be notified + * @throws IllegalStateException in case this method has been already called or Future has been cancelled + */ + public void setReply(final T newValue) { + if (done.getAndSet(true)) { + throw new IllegalStateException("future is not running anymore (either cancelled or already notified)"); + } + this.reply = newValue; + notifyListener(); + } + + public void setException(final Throwable exception) { + if (done.getAndSet(true)) { + throw new IllegalStateException("future is not running anymore (either cancelled or already notified)"); + } + this.exception = exception; + notifyListener(); + } + + private void notifyListener() { + lock.lock(); + try { + processorNotifyCondition.signalAll(); + } finally { + lock.unlock(); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/io/opencmw/utils/LimitedArrayList.java b/core/src/main/java/io/opencmw/utils/LimitedArrayList.java new file mode 100644 index 00000000..abf8e286 --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/LimitedArrayList.java @@ -0,0 +1,54 @@ +package io.opencmw.utils; + +import java.util.ArrayList; + +/** + * @author rstein + * @param generic list element type + */ +public class LimitedArrayList extends ArrayList { + private static final long serialVersionUID = 7158175707385120597L; + private int limit; + + /** + * + * @param limit length of queue in terms of number of elements + */ + public LimitedArrayList(final int limit) { + super(); + if (limit < 1) { + throw new IllegalArgumentException("limit = '" + limit + "'must be >=1 "); + } + this.limit = limit; + } + + @Override + public boolean add(final E o) { + final boolean added = super.add(o); + while (added && size() > limit) { + super.remove(0); + } + return added; + } + + /** + * + * @return length of queue in terms of number of elements + */ + public int getLimit() { + return limit; + } + + /** + * + * @param newLimit length of queue in terms of number of elements + * @return newly set limit (if valid) + */ + public int setLimit(final int newLimit) { + if (newLimit < 1) { + throw new IllegalArgumentException("limit = '" + limit + "'must be >=1 "); + } + limit = newLimit; + return limit; + } +} diff --git a/core/src/main/java/io/opencmw/utils/NoDuplicatesList.java b/core/src/main/java/io/opencmw/utils/NoDuplicatesList.java new file mode 100644 index 00000000..b94faa71 --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/NoDuplicatesList.java @@ -0,0 +1,42 @@ +package io.opencmw.utils; + +import java.util.Collection; +import java.util.LinkedList; + +/** + * @author unknown + * @param generics + */ +public class NoDuplicatesList extends LinkedList { + private static final long serialVersionUID = -8547667608571765668L; + + @Override + public boolean add(E e) { + if (this.contains(e)) { + return false; + } + return super.add(e); + } + + @Override + public void add(int index, E element) { + if (this.contains(element)) { + return; + } + super.add(index, element); + } + + @Override + public boolean addAll(Collection collection) { + Collection copy = new LinkedList<>(collection); + copy.removeAll(this); + return super.addAll(copy); + } + + @Override + public boolean addAll(int index, Collection collection) { + Collection copy = new LinkedList<>(collection); + copy.removeAll(this); + return super.addAll(index, copy); + } +} diff --git a/core/src/main/java/io/opencmw/utils/SharedPointer.java b/core/src/main/java/io/opencmw/utils/SharedPointer.java new file mode 100644 index 00000000..4f2eef3e --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/SharedPointer.java @@ -0,0 +1,82 @@ +package io.opencmw.utils; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +public class SharedPointer { + private static final String CLASS_NAME = SharedPointer.class.getSimpleName().intern(); + private T payload; + private Consumer destroyFunction; + private final AtomicInteger payloadUseCount = new AtomicInteger(0); + + /** + * + * @param payload the raw object to be held by this object + * @return this + */ + public SharedPointer set(final T payload) { + return set(payload, null); + } + + /** + * + * @param payload the raw object to be held by this object + * @param destroyFunction function executed when the last reference is destroyed + * @return this + */ + public SharedPointer set(final T payload, final Consumer destroyFunction) { + assert payload != null : "object must not be null"; + final int usageCount = payloadUseCount.get(); + if (usageCount > 0) { + throw new IllegalStateException("cannot set new variable - object not yet released - usageCount: " + usageCount); + } + this.payload = payload; + this.destroyFunction = destroyFunction; + payloadUseCount.getAndIncrement(); + return this; + } + + public T get() { + return payload; + } + + public R get(Class classType) { + return classType.cast(payload); + } + + public int getReferenceCount() { + return payloadUseCount.get(); + } + + public Class getType() { + return payload == null ? null : payload.getClass(); + } + + /** + * + * @return reference copy of this shared-pointer while increasing the usage count + */ + public SharedPointer getCopy() { + payloadUseCount.getAndIncrement(); + return this; + } + + public void release() { + if (payload == null || payloadUseCount.decrementAndGet() > 0) { + return; + } + if (destroyFunction != null) { + destroyFunction.accept(payload); + } + payload = null; // NOPMD + } + + @Override + public String toString() { + if (payload == null) { + return CLASS_NAME + "[useCount= " + payloadUseCount.get() + ", has destructor=" + (destroyFunction != null) + ", .class, null]"; + } + return CLASS_NAME + "[useCount= " + payloadUseCount.get() + ", has destructor=" + (destroyFunction != null) + ", " // + + payload.getClass().getSimpleName() + ".class, '" + payload.toString() + "']"; + } +} diff --git a/core/src/main/java/io/opencmw/utils/SystemProperties.java b/core/src/main/java/io/opencmw/utils/SystemProperties.java new file mode 100644 index 00000000..a08ec499 --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/SystemProperties.java @@ -0,0 +1,62 @@ +package io.opencmw.utils; + +import static java.util.Map.Entry; + +import java.util.Properties; +import java.util.Set; + +public final class SystemProperties { // NOPMD -- nomen est omen + private static final Properties SYSTEM_PROPERTIES = System.getProperties(); + + private SystemProperties() { + // utility class + } + + public static String getProperty(final String key) { + return SYSTEM_PROPERTIES.getProperty(key); + } + + public static String getPropertyIgnoreCase(String key, String defaultValue) { + String value = SYSTEM_PROPERTIES.getProperty(key); + if (null != value) { + return value; + } + + // Not matching with the actual key then + Set> systemProperties = SYSTEM_PROPERTIES.entrySet(); + for (final Entry entry : systemProperties) { + if (key.equalsIgnoreCase((String) entry.getKey())) { + return (String) entry.getValue(); + } + } + return defaultValue; + } + + public static String getPropertyIgnoreCase(String key) { + return getPropertyIgnoreCase(key, null); + } + + public static double getValue(String key, double defaultValue) { + final String value = getProperty(key); + return value == null ? defaultValue : Double.parseDouble(value); + } + + public static int getValue(String key, int defaultValue) { + final String value = getProperty(key); + return value == null ? defaultValue : Integer.parseInt(value); + } + + public static double getValueIgnoreCase(String key, double defaultValue) { + final String value = getPropertyIgnoreCase(key); + return value == null ? defaultValue : Double.parseDouble(value); + } + + public static int getValueIgnoreCase(String key, int defaultValue) { + final String value = getPropertyIgnoreCase(key); + return value == null ? defaultValue : Integer.parseInt(value); + } + + public static Object put(final Object key, final Object value) { + return SYSTEM_PROPERTIES.put(key, value); + } +} \ No newline at end of file diff --git a/core/src/main/java/io/opencmw/utils/WorkerThreadFactory.java b/core/src/main/java/io/opencmw/utils/WorkerThreadFactory.java new file mode 100644 index 00000000..14de524e --- /dev/null +++ b/core/src/main/java/io/opencmw/utils/WorkerThreadFactory.java @@ -0,0 +1,74 @@ +package io.opencmw.utils; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +import org.jetbrains.annotations.NotNull; + +/** + * OpenCMW thread pool factory and default definitions + * + * default minimum thread pool size is set by system property: 'OpenCMW.defaultPoolSize' + * + * @author rstein + */ +@SuppressWarnings("PMD.DoNotUseThreads") +public class WorkerThreadFactory implements ThreadFactory { + private static final int MAX_THREADS = Math.max(Math.max(4, Runtime.getRuntime().availableProcessors()), // + Integer.parseInt(System.getProperties().getProperty("OpenCMW.defaultPoolSize", "10"))); + private static final AtomicInteger TREAD_COUNTER = new AtomicInteger(); + private static final ThreadFactory DEFAULT_FACTORY = Executors.defaultThreadFactory(); + private static final WorkerThreadFactory SELF = new WorkerThreadFactory("DefaultOpenCmwWorker"); + + private final String poolName; + private final int nThreads; + private final ExecutorService pool; + + public WorkerThreadFactory(final String poolName) { + this(poolName, -1); + } + + public WorkerThreadFactory(final String poolName, final int nThreads) { + this.poolName = poolName; + this.nThreads = nThreads <= 0 ? MAX_THREADS : nThreads; + this.pool = Executors.newFixedThreadPool(this.nThreads, this); + if (this.pool instanceof ThreadPoolExecutor) { + ((ThreadPoolExecutor) pool).setRejectedExecutionHandler((runnable, executor) -> { + try { + // work queue is full -> make the thread calling pool.execute() to wait + executor.getQueue().put(runnable); + } catch (InterruptedException e) { // NOPMD + // silently ignore + } + }); + } + } + + @Override + public Thread newThread(final @NotNull Runnable r) { + final Thread thread = DEFAULT_FACTORY.newThread(r); + TREAD_COUNTER.incrementAndGet(); + thread.setName(poolName + "#" + TREAD_COUNTER.intValue()); + thread.setDaemon(true); + return thread; + } + + public ExecutorService getPool() { + return pool; + } + + public String getPoolName() { + return poolName; + } + + public static WorkerThreadFactory getInstance() { + return SELF; + } + + public int getNumbersOfThreads() { + return nThreads; + } +} diff --git a/core/src/main/java/module-info.java b/core/src/main/java/module-info.java new file mode 100644 index 00000000..95d106d1 --- /dev/null +++ b/core/src/main/java/module-info.java @@ -0,0 +1,15 @@ +open module io.opencmw { + requires disruptor; + requires org.slf4j; + requires jeromq; + requires io.opencmw.serialiser; + requires org.apache.commons.lang3; + requires org.jetbrains.annotations; + requires jsoniter; + + exports io.opencmw; + exports io.opencmw.domain; + exports io.opencmw.filter; + exports io.opencmw.rbac; + exports io.opencmw.utils; +} \ No newline at end of file diff --git a/core/src/test/java/io/opencmw/AggregateEventHandlerTestSource.java b/core/src/test/java/io/opencmw/AggregateEventHandlerTestSource.java new file mode 100644 index 00000000..fd4b4e32 --- /dev/null +++ b/core/src/test/java/io/opencmw/AggregateEventHandlerTestSource.java @@ -0,0 +1,153 @@ +package io.opencmw; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.utils.SharedPointer; + +import com.lmax.disruptor.RingBuffer; + +/** + * An event Source to generate Events with different timing characteristics/orderings. + * + * @author Alexander Krimm + */ +public class AggregateEventHandlerTestSource implements Runnable { + private static final int DEFAULT_CHAIN = 3; + private static final long DEFAULT_DELTA = 20; + private static final long DEFAULT_PAUSE = 400; + // state for the event source + public final int repeat; + public final String[] eventList; + private final RingBuffer ringBuffer; + + /** + * Generate an event source which plays back the given sequence of events + * + * @param events A string containing a space separated list of events. first letter is type/bpcts, second is number + * Optionally you can add semicolon delimited key=value pairs to assign values in of the events + * @param repeat How often to repeat the given sequence (use zero value for infinite repetition) + * @param rb The ring buffer to publish the event into + */ + public AggregateEventHandlerTestSource(final String events, final int repeat, final RingBuffer rb) { + eventList = events.split(" "); + this.repeat = repeat; + this.ringBuffer = rb; + } + + @Override + public void run() { + long lastEvent = System.currentTimeMillis(); + long timeOffset = 0; + int repetitionCount = 0; + while (repeat == 0 || repeat > repetitionCount) { + final Iterator eventIterator = Arrays.stream(eventList).iterator(); + while (!Thread.interrupted() && eventIterator.hasNext()) { + final String eventToken = eventIterator.next(); + final String[] tokens = eventToken.split(";"); + if (tokens.length == 0 || tokens[0].isEmpty()) + continue; + if (tokens[0].equals("pause")) { + lastEvent += DEFAULT_PAUSE; + continue; + } + final Event currentEvent = generateEventFromToken(tokens, timeOffset, lastEvent, repetitionCount); + lastEvent = currentEvent.publishTime; + final long diffMilis = currentEvent.publishTime - System.currentTimeMillis(); + if (diffMilis > 0) { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(diffMilis)); + } + ringBuffer.publishEvent((event, sequence, userPayload) -> { + final TimingCtx ctx = event.getFilter(TimingCtx.class); + if (ctx == null) { + throw new IllegalStateException("RingBufferEvent has not TimingCtx definition"); + } + final EvtTypeFilter evtType = event.getFilter(EvtTypeFilter.class); + if (evtType == null) { + throw new IllegalStateException("RingBufferEvent has not EvtTypeFilter definition"); + } + + event.arrivalTimeStamp = System.currentTimeMillis(); + event.payload = new SharedPointer<>(); + event.payload.set(userPayload.payload); + ctx.setSelector("FAIR.SELECTOR.c=" + userPayload.chain + "", userPayload.bpcts); + evtType.typeName = userPayload.device; + evtType.evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evtType.updateType = EvtTypeFilter.UpdateType.COMPLETE; + }, currentEvent); + } + repetitionCount++; + } + } + + private Event generateEventFromToken(final String[] tokens, final long timeOffset, final long lastEvent, final int repetitionCount) { + String device = tokens[0].substring(0, 1); + long bpcts = Long.parseLong(tokens[0].substring(1)) + repetitionCount * 1000L; + String payload = device + bpcts; + long sourceTime = lastEvent + DEFAULT_DELTA; + long publishTime = sourceTime; + int chain = DEFAULT_CHAIN; + for (int i = 1; i < tokens.length; i++) { + String[] keyvalue = tokens[i].split("="); + if (keyvalue.length != 2) + continue; + switch (keyvalue[0]) { + case "time": + sourceTime = Long.parseLong(keyvalue[1]) + timeOffset; + publishTime = sourceTime; + break; + case "sourceTime": + sourceTime = Long.parseLong(keyvalue[1]) + timeOffset; + break; + case "publishTime": + publishTime = Long.parseLong(keyvalue[1]) + timeOffset; + break; + case "bpcts": + bpcts = Long.parseLong(keyvalue[1]) + repetitionCount * 1000L; + break; + case "chain": + chain = Integer.parseInt(keyvalue[1]); + break; + case "device": + device = keyvalue[1]; + break; + case "payload": + payload = keyvalue[1] + "(repetition count: " + repetitionCount + ")"; + break; + default: + throw new IllegalArgumentException("unable to process event keyvalue pair: " + Arrays.toString(keyvalue)); + } + } + return new Event(sourceTime, publishTime, bpcts, chain, device, payload); + } + + /** + * Mock event entry. + */ + public static class Event { + public final long sourceTime; + public final long publishTime; + public final long bpcts; + public final int chain; + public final String device; + public final Object payload; + + public Event(final long sourceTime, final long publishTime, final long bpcts, final int chain, final String device, final Object payload) { + this.sourceTime = sourceTime; + this.publishTime = publishTime; + this.bpcts = bpcts; + this.chain = chain; + this.device = device; + this.payload = payload; + } + + @Override + public String toString() { + return "Event{sourceTime=" + sourceTime + ", publishTime=" + publishTime + ", bpcts=" + bpcts + ", chain=" + chain + ", device='" + device + '\'' + ", payload=" + payload + '}'; + } + } +} diff --git a/core/src/test/java/io/opencmw/AggregateEventHandlerTests.java b/core/src/test/java/io/opencmw/AggregateEventHandlerTests.java new file mode 100644 index 00000000..97591244 --- /dev/null +++ b/core/src/test/java/io/opencmw/AggregateEventHandlerTests.java @@ -0,0 +1,179 @@ +package io.opencmw; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.awaitility.Awaitility; +import org.hamcrest.Matcher; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.utils.SharedPointer; + +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.TimeoutBlockingWaitStrategy; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.EventHandlerGroup; +import com.lmax.disruptor.dsl.ProducerType; +import com.lmax.disruptor.util.DaemonThreadFactory; + +/** + * Unit-Test for {@link AggregateEventHandler} + * + * @author Alexander Krimm + * @author rstein + */ +class AggregateEventHandlerTests { + private static final Logger LOGGER = LoggerFactory.getLogger(AggregateEventHandlerTests.class); + private static final String[] DEVICES = { "a", "b", "c" }; + private static Stream workingEventSamplesProvider() { // NOPMD NOSONAR - false-positive PMD error (unused function), function is used via reflection + return Stream.of( + arguments("ordinary", "a1 b1 c1 a2 b2 c2 a3 b3 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("duplicate events", "a1 b1 c1 b1 a2 b2 c2 a2 a3 b3 c3 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("reordered", "a1 c1 b1 a2 b2 c2 a3 b3 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("interleaved", "a1 b1 a2 b2 c1 a3 b3 c2 c3", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("missing event", "a1 b1 a2 b2 c2 a3 b3 c3", "a1 b1; a2 b2 c2; a3 b3 c3", "1", 1), + arguments("missing device", "a1 b1 a2 b2 a3 b3", "a1 b1; a2 b2; a3 b3", "1 2 3", 1), + arguments("late", "a1 b1 a2 b2 c2 a3 b3 c3 c1", "a1 b1 c1; a2 b2 c2; a3 b3 c3", "", 1), + arguments("timeout without event", "a1 b1 c1 a2 b2", "a1 b1 c1; a2 b2", "2", 1), + arguments("long queue", "a1 b1 c1 a2 b2", "a1 b1 c1; a2 b2; a1001 b1001 c1001; a1002 b1002; a2001 b2001 c2001; a2002 b2002; a3001 b3001 c3001; a3002 b3002; a4001 b4001 c4001; a4002 b4002", "2 1002 2002 3002 4002", 5), + arguments("simple broken long queue", "a1 b1", "a1 b1; a1001 b1001; a2001 b2001; a3001 b3001; a4001 b4001", "1 1001 2001 3001 4001", 5), + arguments("single event timeout", "a1 b1 pause pause c1", "a1 b1", "1", 1)); + } + + @Test + void testFactoryMethods() { + assertDoesNotThrow(AggregateEventHandler::getFactory); + + final AggregateEventHandler.AggregateEventHandlerFactory factory = AggregateEventHandler.getFactory(); + assertThrows(IllegalArgumentException.class, factory::build); // missing aggregate name + + factory.setAggregateName("testAggregate"); + assertEquals("testAggregate", factory.getAggregateName()); + + assertTrue(factory.getDeviceList().isEmpty()); + factory.setDeviceList("testDevice1", "testDevice2"); + assertEquals(List.of("testDevice1", "testDevice2"), factory.getDeviceList()); + factory.getDeviceList().clear(); + assertTrue(factory.getDeviceList().isEmpty()); + factory.setDeviceList(List.of("testDevice1", "testDevice2")); + assertEquals(List.of("testDevice1", "testDevice2"), factory.getDeviceList()); + factory.getDeviceList().clear(); + + Predicate filter1 = rbEvt -> true; + Predicate filter2 = rbEvt -> false; + assertTrue(factory.getEvtTypeFilter().isEmpty()); + factory.setEvtTypeFilter(filter1, filter2); + assertEquals(List.of(filter1, filter2), factory.getEvtTypeFilter()); + factory.getEvtTypeFilter().clear(); + assertTrue(factory.getEvtTypeFilter().isEmpty()); + factory.setEvtTypeFilter(List.of(filter1, filter2)); + assertEquals(List.of(filter1, filter2), factory.getEvtTypeFilter()); + factory.getEvtTypeFilter().clear(); + + factory.setNumberWorkers(42); + assertEquals(42, factory.getNumberWorkers()); + assertThrows(IllegalArgumentException.class, () -> factory.setNumberWorkers(0)); + + factory.setRetentionSize(13); + assertEquals(13, factory.getRetentionSize()); + assertThrows(IllegalArgumentException.class, () -> factory.setRetentionSize(0)); + + factory.setTimeOut(2, TimeUnit.SECONDS); + assertEquals(2, factory.getTimeOut()); + assertEquals(TimeUnit.SECONDS, factory.getTimeOutUnit()); + assertThrows(IllegalArgumentException.class, () -> factory.setTimeOut(0, TimeUnit.SECONDS)); + assertThrows(IllegalArgumentException.class, () -> factory.setTimeOut(1, null)); + + assertThrows(IllegalArgumentException.class, factory::build); // missing ring buffer assignment + RingBuffer ringBuffer = RingBuffer.createSingleProducer(RingBufferEvent::new, 8, new BlockingWaitStrategy()); + assertThrows(IllegalArgumentException.class, () -> factory.setRingBuffer(null)); + factory.setRingBuffer(ringBuffer); + assertEquals(ringBuffer, factory.getRingBuffer()); + + assertDoesNotThrow(factory::build); + } + + @ParameterizedTest + @MethodSource("workingEventSamplesProvider") + void testSimpleEvents(final String eventSetupName, final String events, final String aggregatesAll, final String timeoutsAll, final int repeat) { + // handler which collects all aggregate events which are republished to the buffer + final Set> aggResults = ConcurrentHashMap.newKeySet(); + final Set aggTimeouts = ConcurrentHashMap.newKeySet(); + + EventHandler testHandler = (evt, seq, eob) -> { + LOGGER.atDebug().addArgument(evt).log("testHandler: {}"); + if (evt.payload == null) { + throw new IllegalStateException("RingBufferEvent payload is null."); + } + + final EvtTypeFilter evtType = evt.getFilter(EvtTypeFilter.class); + if (evtType == null) { + throw new IllegalStateException("RingBufferEvent has no EvtTypeFilter definition"); + } + final TimingCtx timingCtx = evt.getFilter(TimingCtx.class); + if (timingCtx == null) { + throw new IllegalStateException("RingBufferEvent has no TimingCtx definition"); + } + + if (evtType.evtType == EvtTypeFilter.DataType.AGGREGATE_DATA) { + if (!(evt.payload.get() instanceof Map)) { + throw new IllegalStateException("RingBufferEvent payload is not a Map: " + evt.payload.get()); + } + @SuppressWarnings("unchecked") + final Map> agg = (Map>) evt.payload.get(); + final Set payloads = agg.values().stream().map(SharedPointer::get).collect(Collectors.toSet()); + // add aggregate events to result vector + aggResults.add(payloads); + } + + if (evtType.evtType == EvtTypeFilter.DataType.AGGREGATE_DATA && evtType.updateType != EvtTypeFilter.UpdateType.COMPLETE) { + // time-out occured -- add failed aggregate event BPCTS to result vector + aggTimeouts.add(timingCtx.bpcts); + } + }; + + // create event ring buffer and add de-multiplexing processors + final Disruptor disruptor = new Disruptor<>(() -> new RingBufferEvent(TimingCtx.class, EvtTypeFilter.class), 256, DaemonThreadFactory.INSTANCE, ProducerType.MULTI, new TimeoutBlockingWaitStrategy(200, TimeUnit.MILLISECONDS)); + final AggregateEventHandler aggProc = AggregateEventHandler.getFactory().setRingBuffer(disruptor.getRingBuffer()).setAggregateName("testAggregate").setDeviceList(DEVICES).build(); + final EventHandlerGroup endBarrier = disruptor.handleEventsWith(testHandler).handleEventsWith(aggProc).then(aggProc.getAggregationHander()); + RingBuffer rb = disruptor.start(); + + // Use event source to publish demo events to the buffer. + AggregateEventHandlerTestSource testEventSource = new AggregateEventHandlerTestSource(events, repeat, rb); + assertDoesNotThrow(testEventSource::run); + + // wait for all events to be played and processed + Awaitility.await().atMost(Duration.ofSeconds(repeat)).until(() -> endBarrier.asSequenceBarrier().getCursor() == rb.getCursor() && Arrays.stream(aggProc.getAggregationHander()).allMatch(w -> w.bpcts == -1)); + // compare aggregated results and timeouts + assertFalse(aggResults.isEmpty()); + //noinspection unchecked + assertThat(aggResults, containsInAnyOrder(Arrays.stream(aggregatesAll.split(";")) + .filter(s -> !s.isEmpty()) + .map(s -> containsInAnyOrder(Arrays.stream(s.split(" ")).map(String::trim).filter(e -> !e.isEmpty()).toArray())) + .toArray(Matcher[] ::new))); + LOGGER.atDebug().addArgument(aggTimeouts).log("aggTimeouts: {}"); + assertThat(aggTimeouts, containsInAnyOrder(Arrays.stream(timeoutsAll.split(" ")).filter(s -> !s.isEmpty()).map(Long::parseLong).toArray(Long[] ::new))); + } +} diff --git a/core/src/test/java/io/opencmw/EventStoreTest.java b/core/src/test/java/io/opencmw/EventStoreTest.java new file mode 100644 index 00000000..34be287f --- /dev/null +++ b/core/src/test/java/io/opencmw/EventStoreTest.java @@ -0,0 +1,352 @@ +package io.opencmw; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.spi.LoggingEventBuilder; + +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.utils.Cache; +import io.opencmw.utils.SharedPointer; + +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.LifecycleAware; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.TimeoutHandler; +import com.lmax.disruptor.WaitStrategy; +import com.lmax.disruptor.YieldingWaitStrategy; +import com.lmax.disruptor.dsl.Disruptor; + +class EventStoreTest { + private final static Logger LOGGER = LoggerFactory.getLogger(EventStoreTest.class); + private final static boolean IN_ORDER = true; + + @Test + void testFactory() { + assertDoesNotThrow(EventStore::getFactory); + + final EventStore.EventStoreFactory factory = EventStore.getFactory(); + assertDoesNotThrow(factory::build); + + assertEquals(0, factory.getFilterConfig().length); + factory.setFilterConfig(TimingCtx.class, EvtTypeFilter.class); + assertArrayEquals(new Class[] { TimingCtx.class, EvtTypeFilter.class }, factory.getFilterConfig()); + + factory.setLengthHistoryBuffer(42); + assertEquals(42, factory.getLengthHistoryBuffer()); + + factory.setMaxThreadNumber(7); + assertEquals(7, factory.getMaxThreadNumber()); + + factory.setRingbufferSize(128); + assertEquals(128, factory.getRingbufferSize()); + factory.setRingbufferSize(42); + assertEquals(64, factory.getRingbufferSize()); + + factory.setSingleProducer(true); + assertTrue(factory.isSingleProducer()); + factory.setSingleProducer(false); + assertFalse(factory.isSingleProducer()); + + final Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid; + assertNotEquals(muxCtx, factory.getMuxCtxFunction()); + factory.setMuxCtxFunction(muxCtx); + assertEquals(muxCtx, factory.getMuxCtxFunction()); + + final Cache.CacheBuilder> ctxCacheBuilder = Cache.>builder().withLimit(100); + final WaitStrategy customWaitStrategy = new YieldingWaitStrategy(); + assertNotEquals(ctxCacheBuilder, factory.getMuxBuilder()); + assertNotEquals(customWaitStrategy, factory.getWaitStrategy()); + factory.setWaitStrategy(customWaitStrategy).setMuxBuilder(ctxCacheBuilder); + assertEquals(customWaitStrategy, factory.getWaitStrategy()); + assertEquals(ctxCacheBuilder, factory.getMuxBuilder()); + } + + @Test + void basicTest() { + assertDoesNotThrow(() -> EventStore.getFactory().setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build()); + + // global multiplexing context function -> generate new EventStream per detected context, here: multiplexed on {@see TimingCtx#cid} + final Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid; + final Cache.CacheBuilder> ctxCacheBuilder = Cache.>builder().withLimit(100); + final EventStore es = EventStore.getFactory().setMuxCtxFunction(muxCtx).setMuxBuilder(ctxCacheBuilder).setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + Predicate filterBp1 = evt -> evt.test(TimingCtx.class, TimingCtx.matches(-1, -1, 1)); + + es.register(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 1.0: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 1.1: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 1.2: received cid == 1 : payload = {}"); + return null; + }).then(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 2.0: received cid == 1 : payload = {}"); + return null; + }).then(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 3.0: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 3.1: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atTrace().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 3.2: received cid == 1 : payload = {}"); + return null; + }); + + Predicate filterBp0 = evt -> evt.test(TimingCtx.class, TimingCtx.matches(-1, -1, 0)); + es.register(filterBp0, muxCtx, (evts, evtStore, seq, eob) -> { + final String history = evts.stream().map(b -> (String) b.payload.get()).collect(Collectors.joining(", ", "(", ")")); + final String historyAlt = es.getHistory(muxCtx.apply(evts.get(0)), filterBp0, seq, 30).stream().map(b -> (String) b.payload.get()).collect(Collectors.joining(", ", "(", ")")); + LOGGER.atTrace().addArgument(history).log("@@@EventHandler with history: {}"); + + // check identity between the two reference implementations + assertEquals(evts.size(), es.getHistory(muxCtx.apply(evts.get(0)), filterBp0, seq, 30).size()); + assertEquals(history, historyAlt); + return null; + }); + + assertNotNull(es.getRingBuffer()); + + es.start(); + testPublish(es, LOGGER.atTrace(), "message A", 0); + testPublish(es, LOGGER.atTrace(), "message B", 0); + testPublish(es, LOGGER.atTrace(), "message C", 0); + testPublish(es, LOGGER.atTrace(), "message A", 1); + testPublish(es, LOGGER.atTrace(), "message D", 0); + testPublish(es, LOGGER.atTrace(), "message E", 0); + + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); // give a bit of time until all workers are finished + es.stop(); + } + + @Test + void attachEventHandlerTest() { + final Cache.CacheBuilder> ctxCacheBuilder = Cache.>builder().withLimit(100); + final Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid; + final EventStore es = EventStore.getFactory().setMuxCtxFunction(muxCtx).setMuxBuilder(ctxCacheBuilder).setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + final AtomicInteger handlerCount1 = new AtomicInteger(); + final AtomicInteger handlerCount2 = new AtomicInteger(); + final AtomicInteger handlerCount3 = new AtomicInteger(); + es.register((evt, seq, eob) -> handlerCount1.incrementAndGet()) + .and((evt, seq, eob) -> handlerCount2.incrementAndGet()) + .then((evt, seq, eob) -> { + assertTrue(handlerCount1.get() >= handlerCount3.get()); + assertTrue(handlerCount2.get() >= handlerCount3.get()); + handlerCount3.incrementAndGet(); + }); + + es.start(); + testPublish(es, LOGGER.atTrace(), "A", 0); + testPublish(es, LOGGER.atTrace(), "B", 0); + testPublish(es, LOGGER.atTrace(), "C", 0); + testPublish(es, LOGGER.atTrace(), "A", 1); + testPublish(es, LOGGER.atTrace(), "D", 0); + testPublish(es, LOGGER.atTrace(), "E", 0); + Awaitility.await().atMost(200, TimeUnit.MILLISECONDS).until(() -> { + es.stop(); + return true; }); + + assertEquals(6, handlerCount1.get()); + assertEquals(6, handlerCount2.get()); + assertEquals(6, handlerCount3.get()); + } + + @Test + void attachHistoryEventHandlerTest() { + final Cache.CacheBuilder> ctxCacheBuilder = Cache.>builder().withLimit(100); + final Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid; + final EventStore es = EventStore.getFactory().setMuxCtxFunction(muxCtx).setMuxBuilder(ctxCacheBuilder).setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + final AtomicInteger handlerCount1 = new AtomicInteger(); + final AtomicInteger handlerCount2 = new AtomicInteger(); + final AtomicInteger handlerCount3 = new AtomicInteger(); + Predicate filterBp1 = evt -> evt.test(TimingCtx.class, TimingCtx.matches(-1, -1, 0)); + es.register(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + handlerCount1.incrementAndGet(); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + handlerCount2.incrementAndGet(); + return null; + }).then(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + assertTrue(handlerCount1.get() >= handlerCount3.get()); + assertTrue(handlerCount2.get() >= handlerCount3.get()); + handlerCount3.incrementAndGet(); + return null; + }); + + es.start(); + testPublish(es, LOGGER.atTrace(), "A", 0); + testPublish(es, LOGGER.atTrace(), "B", 0); + testPublish(es, LOGGER.atTrace(), "C", 0); + testPublish(es, LOGGER.atTrace(), "A", 1); + testPublish(es, LOGGER.atTrace(), "D", 0); + testPublish(es, LOGGER.atTrace(), "E", 0); + Awaitility.await().atMost(200, TimeUnit.MILLISECONDS).until(() -> { + es.stop(); + return true; }); + + assertEquals(5, handlerCount1.get()); + assertEquals(5, handlerCount2.get()); + assertEquals(5, handlerCount3.get()); + } + + @Test + void historyTest() { + final Cache.CacheBuilder> ctxCacheBuilder = Cache.>builder().withLimit(100); + final Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid; + final EventStore es = EventStore.getFactory().setMuxCtxFunction(muxCtx).setMuxBuilder(ctxCacheBuilder).setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + es.start(); + testPublish(es, LOGGER.atTrace(), "A", 0); + testPublish(es, LOGGER.atTrace(), "B", 0); + testPublish(es, LOGGER.atTrace(), "C", 0); + testPublish(es, LOGGER.atTrace(), "A", 1); + testPublish(es, LOGGER.atTrace(), "D", 0); + testPublish(es, LOGGER.atTrace(), "E", 0); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); // give a bit of time until all workers are finished + + final Optional lastEvent = es.getLast("cid=0", evt -> true); + assertTrue(lastEvent.isPresent()); + assertFalse(es.getLast("cid=0", evt -> evt.matches(TimingCtx.class, TimingCtx.matches(-1, -1, 2))).isPresent()); + LOGGER.atTrace().addArgument(lastEvent.get().payload.get(String.class)).log("retrieved last event = {}"); + + final List eventHistory = es.getHistory("cid=0", evt -> evt.matches(String.class) && evt.matches(TimingCtx.class, TimingCtx.matches(-1, -1, 0)), 4); + final String history = eventHistory.stream().map(b -> b.payload.get(String.class)).collect(Collectors.joining(", ", "(", ")")); + LOGGER.atTrace().addArgument(history).log("retrieved last events = {}"); + + es.stop(); + } + + public static void main(final String[] args) { + // global multiplexing context function -> generate new EventStream per detected context, here: multiplexed on {@see TimingCtx#cid} + final Function muxCtx = evt -> "cid=" + evt.getFilter(TimingCtx.class).cid; + final Cache.CacheBuilder> ctxCacheBuilder = Cache.>builder().withLimit(100); + final EventStore es = EventStore.getFactory().setMuxCtxFunction(muxCtx).setMuxBuilder(ctxCacheBuilder).setFilterConfig(TimingCtx.class, EvtTypeFilter.class).build(); + + final MyHandler handler1 = new MyHandler("Handler1", es.getRingBuffer()); + MyHandler handler2 = new MyHandler("Handler2", es.getRingBuffer()); + EventHandler lambdaEventHandler = (evt, seq, buffer) -> // + LOGGER.atInfo().addArgument(seq).addArgument(evt.payload.get()).addArgument(es.getRingBuffer().getMinimumGatingSequence()).log("Lambda-Handler3 seq:{} - '{}' - gate ='{}'"); + + if (IN_ORDER) { + // execute in order + es.register(handler1).then(handler2).then(lambdaEventHandler); + } else { + //execute out-of-order + es.register(handler1).and(handler2).and(lambdaEventHandler); + } + + Predicate filterBp1 = evt -> evt.test(TimingCtx.class, TimingCtx.matches(-1, -1, 1)); + es.register(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 1.0: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 1.1: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 1.2: received cid == 1 : payload = {}"); + return null; + }).then(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 2.0: received cid == 1 : payload = {}"); + return null; + }).then(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 3.0: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 3.1: received cid == 1 : payload = {}"); + return null; + }).and(filterBp1, muxCtx, (evts, evtStore, seq, eob) -> { + LOGGER.atInfo().addArgument(evts.get(0).payload.get()).log("SequencedFilteredTask 3.2: received cid == 1 : payload = {}"); + return null; + }); + + EventHandler printEndHandler = (evt, seq, buffer) -> // + LOGGER.atInfo().addArgument(es.getRingBuffer().getMinimumGatingSequence()) // + .addArgument(es.disruptor.getSequenceValueFor(handler1)) // + .addArgument(es.disruptor.getSequenceValueFor(handler2)) // + .addArgument(es.disruptor.getSequenceValueFor(lambdaEventHandler)) // + .addArgument(seq) // + .log("### gating position = {} sequences for handler 1: {} 2: {} 3:{} ph: {}"); + + es.register(printEndHandler); + + Predicate filterBp0 = evt -> evt.test(TimingCtx.class, TimingCtx.matches(-1, -1, 0)); + es.register(filterBp0, muxCtx, (evts, evtStore, seq, eob) -> { + final String history = evts.stream().map(b -> (String) b.payload.get()).collect(Collectors.joining(", ", "(", ")")); + final String historyAlt = es.getHistory(muxCtx.apply(evts.get(0)), filterBp0, seq, 30).stream().map(b -> (String) b.payload.get()).collect(Collectors.joining(", ", "(", ")")); + LOGGER.atInfo().addArgument(history).log("@@@EventHandlerA with history: {}"); + LOGGER.atInfo().addArgument(historyAlt).log("@@@EventHandlerB with history: {}"); + return null; + }); + + es.start(); + + testPublish(es, LOGGER.atInfo(), "message A", 0); + testPublish(es, LOGGER.atInfo(), "message B", 0); + testPublish(es, LOGGER.atInfo(), "message C", 0); + testPublish(es, LOGGER.atInfo(), "message A", 1); + testPublish(es, LOGGER.atInfo(), "message D", 0); + testPublish(es, LOGGER.atInfo(), "message E", 0); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); // give a bit of time until all workers are finished + es.stop(); + } + + protected static void testPublish(EventStore eventStore, final LoggingEventBuilder logger, final String payLoad, final int beamProcess) { + eventStore.getRingBuffer().publishEvent((event, sequence, buffer) -> { + event.arrivalTimeStamp = System.currentTimeMillis() * 1000; + event.parentSequenceNumber = sequence; + event.getFilter(TimingCtx.class).setSelector("FAIR.SELECTOR.C=0:S=0:P=" + beamProcess, event.arrivalTimeStamp); + event.payload = new SharedPointer<>(); + event.payload.set("pid=" + beamProcess + ": " + payLoad); + logger.addArgument(sequence).addArgument(event.payload.get()).addArgument(buffer).log("publish Seq:{} - event:'{}' buffer:'{}'"); + }); + } + + public static class MyHandler implements EventHandler, TimeoutHandler, LifecycleAware { + private final RingBuffer ringBuffer; + + private final String handlerName; + + public MyHandler(final String handlerName, final RingBuffer ringBuffer) { + this.handlerName = handlerName; + this.ringBuffer = ringBuffer; + } + + @Override + public void onEvent(final RingBufferEvent event, final long sequence, final boolean endOfBatch) { + LOGGER.atInfo().addArgument(handlerName).addArgument(sequence).addArgument(event.payload.get()).log("'{}'- process sequence ID: {} event = {}"); + } + + @Override + public void onShutdown() { + LOGGER.atInfo().addArgument(MyHandler.class).addArgument(handlerName).log("stopped '{}'-name:'{}'"); + } + + @Override + public void onStart() { + LOGGER.atInfo().addArgument(MyHandler.class).addArgument(handlerName).log("started '{}'-name:'{}'"); + } + + @Override + public void onTimeout(final long sequence) { + LOGGER.atInfo().addArgument(handlerName).addArgument(sequence).addArgument(ringBuffer.getMinimumGatingSequence()).log("onTimeout '{}'-sequence:'{}' - gate:'{}'"); + } + } +} diff --git a/core/src/test/java/io/opencmw/MimeTypeTests.java b/core/src/test/java/io/opencmw/MimeTypeTests.java new file mode 100644 index 00000000..09dd32d4 --- /dev/null +++ b/core/src/test/java/io/opencmw/MimeTypeTests.java @@ -0,0 +1,85 @@ +package io.opencmw; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +/** + * Tests of {@link io.opencmw.MimeType} + * + * @author rstein + */ +class MimeTypeTests { + @ParameterizedTest + @EnumSource(MimeType.class) + void genericTests(final MimeType mType) { + assertNotNull(mType.toString()); + final String mimeTypeName = "'" + mType + "'"; + + assertEquals(mType, MimeType.getEnum(mType.toString()), "getEnum: mType = " + mimeTypeName); + assertEquals(mType, MimeType.getEnum(mType.name()), "getEnum: mType = " + mimeTypeName); + + assertNotNull(mType.getDescription(), "description: mType = " + mimeTypeName); + assertNotNull(mType.getType(), "type: mType = " + mimeTypeName); + assertNotNull(mType.getSubType(), "subType: mType = " + mimeTypeName); + assertNotNull(mType.getFileEndings(), "fileEndings: mType = " + mimeTypeName); + + if (!mType.getFileEndings().isEmpty()) { + for (String fileName : mType.getFileEndings()) { + if (mType.equals(MimeType.APNG)) { + // skip file-ending tests since PNG and APNG have same/very similar ending .png (and the more rare .apng) + continue; + } + assertEquals(mType, MimeType.getEnumByFileName(fileName), "fileEndings - match: mType = " + mimeTypeName); + } + } + + final String typeRaw = mType.toString().split("/")[0]; + assertTrue(typeRaw.contentEquals(mType.getType().toString()), "mType = " + mimeTypeName); + + final String description = mType.toString(); + if (mType.isImageData()) { + assertTrue(description.startsWith("image"), "image?: mType = " + mimeTypeName); + } else { + assertFalse(description.startsWith("image"), "image?: mType = " + mimeTypeName); + } + + if (mType.isVideoData()) { + assertTrue(description.startsWith("video"), "video?: mType = " + mimeTypeName); + } else { + assertFalse(description.startsWith("video"), "video?: mType = " + mimeTypeName); + } + + if (mType.isTextData()) { + assertTrue(description.startsWith("text"), "text?: mType = " + mimeTypeName); + } else { + assertFalse(description.startsWith("text"), "text?: mType = " + mimeTypeName); + } + + if (mType.isNonDisplayableData()) { + assertFalse(mType.isImageData(), "nonDisplayableData?: mType = " + mimeTypeName); + assertFalse(mType.isVideoData(), "nonDisplayableData?: mType = " + mimeTypeName); + } + } + + @Test + void cornerCaseTests() { + assertEquals(MimeType.UNKNOWN, MimeType.getEnum(null)); + assertEquals(MimeType.UNKNOWN, MimeType.getEnum("")); + assertEquals(MimeType.UNKNOWN, MimeType.getEnum(" ")); + assertEquals(MimeType.UNKNOWN, MimeType.getEnum("video/made-up-format")); + assertEquals(MimeType.UNKNOWN, MimeType.getEnum("wormhole/made-up-format")); + + assertEquals(MimeType.UNKNOWN, MimeType.getEnumByFileName(null)); + assertEquals(MimeType.UNKNOWN, MimeType.getEnumByFileName("")); + assertEquals(MimeType.UNKNOWN, MimeType.getEnumByFileName(" ")); + assertEquals(MimeType.UNKNOWN, MimeType.getEnumByFileName(".xyz42")); + + assertEquals(MimeType.Type.UNKNOWN, MimeType.Type.getEnum(null)); + assertEquals(MimeType.Type.UNKNOWN, MimeType.Type.getEnum("")); + assertEquals(MimeType.Type.UNKNOWN, MimeType.Type.getEnum(" ")); + assertEquals(MimeType.Type.UNKNOWN, MimeType.Type.getEnum("made-up-type")); + } +} diff --git a/core/src/test/java/io/opencmw/OpenCmwProtocolTests.java b/core/src/test/java/io/opencmw/OpenCmwProtocolTests.java new file mode 100644 index 00000000..7b5dd379 --- /dev/null +++ b/core/src/test/java/io/opencmw/OpenCmwProtocolTests.java @@ -0,0 +1,95 @@ +package io.opencmw; + +import static org.junit.jupiter.api.Assertions.*; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +import org.junit.jupiter.api.Test; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +/** + * basic MDP OpenCMW protocol consistency tests + */ +class OpenCmwProtocolTests { + private static final OpenCmwProtocol.MdpMessage TEST_MESSAGE = new OpenCmwProtocol.MdpMessage("senderName".getBytes(StandardCharsets.UTF_8), // + OpenCmwProtocol.MdpSubProtocol.PROT_CLIENT, OpenCmwProtocol.Command.GET_REQUEST, // + "serviceName".getBytes(StandardCharsets.UTF_8), new byte[] { (byte) 3, (byte) 2, (byte) 1 }, + URI.create("test/topic"), "test data - Hello World!".getBytes(StandardCharsets.UTF_8), "error message", "rbacToken".getBytes(StandardCharsets.UTF_8)); + + @Test + void testCommandEnum() { + for (OpenCmwProtocol.Command cmd : OpenCmwProtocol.Command.values()) { + assertEquals(cmd, OpenCmwProtocol.Command.getCommand(cmd.getData())); + assertNotNull(cmd.toString()); + if (!cmd.equals(OpenCmwProtocol.Command.UNKNOWN)) { + assertTrue(cmd.isClientCompatible() || cmd.isWorkerCompatible()); + } + } + assertFalse(OpenCmwProtocol.Command.UNKNOWN.isWorkerCompatible()); + assertFalse(OpenCmwProtocol.Command.UNKNOWN.isClientCompatible()); + } + + @Test + void testMdpSubProtocolEnum() { + for (OpenCmwProtocol.MdpSubProtocol cmd : OpenCmwProtocol.MdpSubProtocol.values()) { + assertEquals(cmd, OpenCmwProtocol.MdpSubProtocol.getProtocol(cmd.getData())); + assertNotNull(cmd.toString()); + } + } + + @Test + void testMdpIdentity() { + final OpenCmwProtocol.MdpMessage test = TEST_MESSAGE; + assertEquals(OpenCmwProtocol.Command.GET_REQUEST, test.command); + assertEquals(TEST_MESSAGE, test, "object identity"); + assertNotEquals(TEST_MESSAGE, new Object(), "inequality if different class type"); + final OpenCmwProtocol.MdpMessage clone = new OpenCmwProtocol.MdpMessage(TEST_MESSAGE); + assertEquals(TEST_MESSAGE, clone, "copy constructor"); + assertEquals(TEST_MESSAGE.hashCode(), clone.hashCode(), "hashCode equality"); + final OpenCmwProtocol.MdpMessage modified = new OpenCmwProtocol.MdpMessage(TEST_MESSAGE); + modified.protocol = OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; + assertNotEquals(TEST_MESSAGE, modified, "copy constructor"); + assertTrue(TEST_MESSAGE.hasRbackToken(), "non-empty rbac token"); + assertEquals("senderName", TEST_MESSAGE.getSenderName(), "sender name string"); + assertEquals("serviceName", TEST_MESSAGE.getServiceName(), "service name string"); + assertNotNull(TEST_MESSAGE.toString()); + } + + @Test + void testMdpSendReceiveIdentity() { + try (ZContext ctx = new ZContext()) { + { + final ZMQ.Socket receiveSocket1 = ctx.createSocket(SocketType.ROUTER); + receiveSocket1.bind("inproc://pair1"); + final ZMQ.Socket receiveSocket2 = ctx.createSocket(SocketType.DEALER); + receiveSocket2.bind("inproc://pair2"); + final ZMQ.Socket sendSocket = ctx.createSocket(SocketType.DEALER); + sendSocket.setIdentity(TEST_MESSAGE.senderID); + sendSocket.connect("inproc://pair1"); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(200)); + + TEST_MESSAGE.send(sendSocket); + final OpenCmwProtocol.MdpMessage reply = OpenCmwProtocol.MdpMessage.receive(receiveSocket1); + assertEquals(TEST_MESSAGE, reply, "serialisation identity via router"); + + sendSocket.disconnect("inproc://pair1"); + sendSocket.connect("inproc://pair2"); + TEST_MESSAGE.send(sendSocket); + final OpenCmwProtocol.MdpMessage reply2 = OpenCmwProtocol.MdpMessage.receive(receiveSocket2); + assertEquals(TEST_MESSAGE, reply2, "serialisation identity via dealer"); + + OpenCmwProtocol.MdpMessage.send(sendSocket, List.of(TEST_MESSAGE)); + final OpenCmwProtocol.MdpMessage reply3 = OpenCmwProtocol.MdpMessage.receive(receiveSocket2); + final OpenCmwProtocol.MdpMessage clone = new OpenCmwProtocol.MdpMessage(TEST_MESSAGE); + clone.command = OpenCmwProtocol.Command.FINAL; // N.B. multiple message exist only for reply type either FINAL, or (PARTIAL, PARTIAL, ..., FINAL) + assertEquals(clone, reply3, "serialisation identity via dealer"); + } + } + } +} diff --git a/core/src/test/java/io/opencmw/QueryParameterParserTest.java b/core/src/test/java/io/opencmw/QueryParameterParserTest.java new file mode 100644 index 00000000..3d87af15 --- /dev/null +++ b/core/src/test/java/io/opencmw/QueryParameterParserTest.java @@ -0,0 +1,226 @@ +package io.opencmw; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import io.opencmw.filter.TimingCtx; + +class QueryParameterParserTest { + @SuppressWarnings("CanBeFinal") + private static class TestQueryClass { + public String param1; + public int param2; + public MimeType mimeType; + public Object specialClass; + public UnknownClass unknownClass; + public TimingCtx ctx; + } + + private static class UnknownClass { + // empty unknown class + } + + @SuppressWarnings("CanBeFinal") + private static class TestQueryClass2 { + protected boolean dummyBoolean; + protected byte dummyByte; + protected short dummyShort; + protected int dummyInt; + protected long dummyLong; + protected float dummyFloat; + protected double dummyDouble; + protected Boolean dummyBoxedBoolean = Boolean.FALSE; + protected Byte dummyBoxedByte = (byte) 0; + protected Short dummyBoxedShort = (short) 0; + protected Integer dummyBoxedInt = 0; + protected Long dummyBoxedLong = 0L; + protected Float dummyBoxedFloat = 0f; + protected Double dummyBoxedDouble = 0.0; + protected String dummyString1 = "nope"; + protected String dummyString2 = "nope"; + protected String dummyString3; + protected String dummyString4; + } + + @Test + void testMapStringToClass() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + URI uri = URI.create("https://opencmw.io?param1=Hello¶m2=42&mimeType=text/html&specialClass"); + final TestQueryClass ctx = QueryParameterParser.parseQueryParameter(TestQueryClass.class, uri.getQuery()); + assertEquals("Hello", ctx.param1); + assertEquals(42, ctx.param2); + assertEquals(MimeType.HTML, ctx.mimeType); + assertNotNull(ctx.specialClass); + + assertThrows(IllegalArgumentException.class, () -> QueryParameterParser.parseQueryParameter(TestQueryClass.class, "param1=b;param2=c")); + } + + @Test + void testMapStringToClassWithMissingParameter() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + URI uri = URI.create("https://opencmw.io?param1=Hello¶m2=42&specialClass"); + final TestQueryClass ctx = QueryParameterParser.parseQueryParameter(TestQueryClass.class, uri.getQuery()); + assertEquals("Hello", ctx.param1); + assertEquals(42, ctx.param2); + assertNull(ctx.mimeType); // was missing in parameters + assertNotNull(ctx.specialClass); + + assertThrows(IllegalArgumentException.class, () -> QueryParameterParser.parseQueryParameter(TestQueryClass.class, "param1=b;param2=c")); + } + + @Test + void testMapClassToString() { + TestQueryClass ctx = new TestQueryClass(); + ctx.param1 = "Hello"; + ctx.param2 = 42; + ctx.mimeType = MimeType.HTML; + ctx.ctx = TimingCtx.get("FAIR.SELECTOR.C=2"); + ctx.specialClass = new Object(); + + String result = QueryParameterParser.generateQueryParameter(ctx); + // System.err.println("result = " + result); + assertNotNull(result); + assertTrue(result.contains(ctx.param1)); + assertTrue(result.contains("" + ctx.param2)); + assertEquals("param1=Hello¶m2=42&mimeType=HTML&specialClass=&unknownClass=&ctx=FAIR.SELECTOR.C%3D2", result); + } + + @Test + void testClassToStringFunctions() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + final String testString = "dummyBoolean=true;dummyBoxedBoolean=true;" + + "dummyByte=2&dummyBoxedByte=2;" + + "dummyShort=3&dummyBoxedShort=3;" + + "dummyInt=4&dummyBoxedInt=4;" + + "dummyLong=5&dummyBoxedLong=5;" + + "dummyFloat=6.&dummyBoxedFloat=6.0;" + + "dummyDouble=7.0&dummyBoxedDouble=7.0;" + + "dummyString1=TestA;dummyString2=\"TestA \";dummyString=;dummyString4=\"equation =%3D5\""; + final TestQueryClass2 ctx = QueryParameterParser.parseQueryParameter(TestQueryClass2.class, testString); + assertTrue(ctx.dummyBoolean); + assertTrue(ctx.dummyBoxedBoolean); + assertEquals((byte) 2, ctx.dummyByte); + assertEquals((byte) 2, ctx.dummyBoxedByte); + assertEquals((short) 3, ctx.dummyShort); + assertEquals((short) 3, ctx.dummyBoxedShort); + assertEquals(4, ctx.dummyInt); + assertEquals(4, ctx.dummyBoxedInt); + assertEquals(5L, ctx.dummyLong); + assertEquals(5L, ctx.dummyBoxedLong); + assertEquals(6.f, ctx.dummyFloat); + assertEquals(6.f, ctx.dummyBoxedFloat); + assertEquals(7., ctx.dummyDouble); + assertEquals(7., ctx.dummyBoxedDouble); + assertEquals("TestA", ctx.dummyString1); + assertEquals("\"TestA \"", ctx.dummyString2); + assertNull(ctx.dummyString3); + assertEquals("\"equation ==5\"", ctx.dummyString4); + } + + @Test + void testMapFunction() { + URI uri = URI.create("https://opencmw.io?param1=value1¶m2=¶m3=value3¶m3"); + final Map> map = QueryParameterParser.getMap(uri.getQuery()); + + assertNotNull(map.get("param1")); + assertEquals(1, map.get("param1").size()); + assertEquals("value1", map.get("param1").get(0)); + + assertNotNull(map.get("param2")); + assertEquals(1, map.get("param2").size()); + assertNull(map.get("param2").get(0)); + + assertNotNull(map.get("param3")); + assertEquals(2, map.get("param3").size()); + assertEquals("value3", map.get("param3").get(0)); + assertNull(map.get("param3").get(1)); + + assertNull(map.get("param4"), "null for non-existent parameter"); + + assertEquals(0, QueryParameterParser.getMap("").size(), "empty map for empty query string"); + } + + @Test + void testMimetype() { + assertEquals(MimeType.HTML, QueryParameterParser.getMimeType("contentType=text/html")); + assertEquals(MimeType.HTML, QueryParameterParser.getMimeType("contentType=HTML")); + assertEquals(MimeType.HTML, QueryParameterParser.getMimeType("contentType=html")); + assertEquals(MimeType.HTML, QueryParameterParser.getMimeType("contentType=text/HtmL")); + } + + @Test + void testIdentity() { + final String queryString = "param1=value1a¶m2=value2¶m3=value3¶m1=value1b"; + final Map> parsedMap = QueryParameterParser.getMap(queryString); + assertEquals(3, parsedMap.size(), "number of unique parameter"); + assertEquals(2, parsedMap.get("param1").size()); + assertEquals(1, parsedMap.get("param2").size()); + assertEquals(1, parsedMap.get("param3").size()); + assertEquals(List.of("value1a", "value1b"), parsedMap.get("param1")); + assertEquals(List.of("value2"), parsedMap.get("param2")); + assertEquals(List.of("value3"), parsedMap.get("param3")); + final Map returnMap = new HashMap<>(parsedMap); + returnMap.put("param4", "value4"); + final String returnString = QueryParameterParser.generateQueryParameter(returnMap); + + // test second generated map + final Map> parsedMap2 = QueryParameterParser.getMap(returnString); + assertEquals(4, parsedMap2.size(), "number of unique parameter"); + assertEquals(2, parsedMap2.get("param1").size()); + assertEquals(1, parsedMap2.get("param2").size()); + assertEquals(1, parsedMap2.get("param3").size()); + assertEquals(1, parsedMap2.get("param4").size()); + assertEquals(List.of("value1a", "value1b"), parsedMap2.get("param1")); + assertEquals(List.of("value2"), parsedMap2.get("param2")); + assertEquals(List.of("value3"), parsedMap2.get("param3")); + assertEquals(List.of("value4"), parsedMap2.get("param4")); + + // generate special cases + final Map testMap1 = new HashMap<>(); + testMap1.put("param1", null); + assertEquals("param1", QueryParameterParser.generateQueryParameter(testMap1)); + testMap1.put("param2", null); + assertEquals("param1¶m2", QueryParameterParser.generateQueryParameter(testMap1)); + testMap1.put("param3", Arrays.asList(null, null)); + assertEquals("param3¶m3¶m1¶m2", QueryParameterParser.generateQueryParameter(testMap1)); + } + + @Test + void testAddQueryParameter() throws URISyntaxException { + final URI baseUri = URI.create("basePath"); + final URI extUri = URI.create("basePath?param1"); + assertEquals(baseUri, QueryParameterParser.appendQueryParameter(baseUri, null)); + assertEquals(baseUri, QueryParameterParser.appendQueryParameter(baseUri, "")); + assertEquals(URI.create("basePath?test"), QueryParameterParser.appendQueryParameter(baseUri, "test")); + assertEquals(URI.create("basePath?param1&test"), QueryParameterParser.appendQueryParameter(extUri, "test")); + } + + @Test + void testRemoveQueryParameter() throws URISyntaxException { + final URI origURI = URI.create("basePath?param1=value1a¶m2=value2¶m3=value3¶m1=value1b"); + + assertEquals(origURI, QueryParameterParser.removeQueryParameter(origURI, null)); + assertEquals(origURI, QueryParameterParser.removeQueryParameter(origURI, "")); + assertEquals(URI.create("basePath"), QueryParameterParser.removeQueryParameter(URI.create("basePath"), "bla")); + + // remove 'param1' with specific value 'value1a' - > value1b should remain + final Map> reference1 = QueryParameterParser.getMap("param2=value2¶m3=value3¶m1=value1b"); + final Map> result1 = QueryParameterParser.getMap(QueryParameterParser.removeQueryParameter(origURI, "param1=value1a").getQuery()); + assertEquals(reference1, result1); + + // remove all 'param1' by key, only 'param2' and 'param3' should remain + final Map> reference2 = QueryParameterParser.getMap("param2=value2¶m3=value3"); + final Map> result2 = QueryParameterParser.getMap(QueryParameterParser.removeQueryParameter(origURI, "param1").getQuery()); + assertEquals(reference2, result2); + + // remove 'param1' with specific value 'value1a' and then 'value1b' - > only 'param2' and 'param3' should remain + final Map> result3 = QueryParameterParser.getMap(QueryParameterParser.removeQueryParameter(QueryParameterParser.removeQueryParameter(origURI, "param1=value1a"), "param1=value1b").getQuery()); + assertEquals(reference2, result3); + } +} diff --git a/core/src/test/java/io/opencmw/RingBufferEventTests.java b/core/src/test/java/io/opencmw/RingBufferEventTests.java new file mode 100644 index 00000000..9b3e5fdf --- /dev/null +++ b/core/src/test/java/io/opencmw/RingBufferEventTests.java @@ -0,0 +1,135 @@ +package io.opencmw; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +import io.opencmw.filter.EvtTypeFilter; +import io.opencmw.filter.TimingCtx; +import io.opencmw.utils.SharedPointer; + +class RingBufferEventTests { + @Test + void basicTests() { + assertDoesNotThrow(() -> new RingBufferEvent(TimingCtx.class)); + assertThrows(IllegalArgumentException.class, () -> new RingBufferEvent(TimingCtx.class, BogusFilter.class)); + + final RingBufferEvent evt = new RingBufferEvent(TimingCtx.class); + assertFalse(evt.matches(String.class)); + evt.payload = new SharedPointer<>(); + assertFalse(evt.matches(String.class)); + evt.payload.set("Hello World"); + assertTrue(evt.matches(String.class)); + evt.throwables.add(new Throwable("test")); + assertNotNull(evt.toString()); + + // assert copy/clone interfaces + assertEquals(evt, evt.clone()); + final RingBufferEvent evt2 = new RingBufferEvent(TimingCtx.class); + evt.copyTo(evt2); + assertEquals(evt, evt2); + + assertDoesNotThrow(evt::clear); + assertEquals(0, evt.throwables.size()); + assertEquals(0, evt.arrivalTimeStamp); + + final long timeNowMicros = System.currentTimeMillis() * 1000; + final TimingCtx ctxFilter = evt.getFilter(TimingCtx.class); + assertNotNull(ctxFilter); + assertThrows(IllegalArgumentException.class, () -> evt.getFilter(BogusFilter.class)); + + ctxFilter.setSelector("FAIR.SELECTOR.C=3:S=2", timeNowMicros); + + // assert copy/clone interfaces for cleared evt + evt.clear(); + assertEquals(evt, evt.clone()); + final RingBufferEvent evt3 = new RingBufferEvent(TimingCtx.class); + evt.copyTo(evt3); + assertEquals(evt, evt3); + } + + @Test + void basicUsageTests() { + final RingBufferEvent evt = new RingBufferEvent(TimingCtx.class, EvtTypeFilter.class); + assertNotNull(evt); + final long timeNowMicros = System.currentTimeMillis() * 1000; + evt.arrivalTimeStamp = timeNowMicros; + evt.getFilter(EvtTypeFilter.class).evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evt.getFilter(EvtTypeFilter.class).typeName = "MyDevice"; + evt.getFilter(TimingCtx.class).setSelector("FAIR.SELECTOR.C=3:S=2", timeNowMicros); + + evt.matches(TimingCtx.class, ctx -> { + System.err.println("received ctx = " + ctx); + return true; + }); + + // fall-back filter: the whole RingBufferEvent, all Filters etc are accessible + assertTrue(evt.matches(e -> e.arrivalTimeStamp == timeNowMicros)); + + // filter only on given filter trait - here TimingCtx + assertTrue(evt.matches(TimingCtx.class, TimingCtx.matches(3, 2))); + evt.test(TimingCtx.class, TimingCtx.matches(3, 2)); + + // combination of filter traits + assertTrue(evt.test(TimingCtx.class, TimingCtx.matches(3, 2)) && evt.test(EvtTypeFilter.class, dataType -> dataType.evtType == EvtTypeFilter.DataType.DEVICE_DATA)); + assertTrue(evt.test(TimingCtx.class, TimingCtx.matches(3, 2)) && evt.test(EvtTypeFilter.class, EvtTypeFilter.isDeviceData("MyDevice"))); + assertTrue(evt.test(TimingCtx.class, TimingCtx.matches(3, 2).and(TimingCtx.isNewerBpcts(timeNowMicros - 1L)))); + } + + @Test + void equalsTests() { + final RingBufferEvent evt1 = new RingBufferEvent(TimingCtx.class); + final RingBufferEvent evt2 = new RingBufferEvent(TimingCtx.class); + + assertEquals(evt1, evt1, "equals identity"); + assertNotEquals(null, evt1, "equals null"); + evt1.parentSequenceNumber = 42; + assertNotEquals(evt1.hashCode(), evt2.hashCode(), "equals hashCode"); + assertNotEquals(evt1, evt2, "equals hashCode"); + } + + @Test + void testClearEventHandler() { + final RingBufferEvent evt = new RingBufferEvent(TimingCtx.class, EvtTypeFilter.class); + assertNotNull(evt); + final long timeNowMicros = System.currentTimeMillis() * 1000; + evt.arrivalTimeStamp = timeNowMicros; + + assertEquals(timeNowMicros, evt.arrivalTimeStamp); + assertDoesNotThrow(RingBufferEvent.ClearEventHandler::new); + + final RingBufferEvent.ClearEventHandler clearHandler = new RingBufferEvent.ClearEventHandler(); + assertNotNull(clearHandler); + + clearHandler.onEvent(evt, 0, false); + assertEquals(0, evt.arrivalTimeStamp); + } + + @Test + void testHelper() { + assertNotNull(RingBufferEvent.getPrintableStackTrace(new Throwable("pretty print"))); + assertNotNull(RingBufferEvent.getPrintableStackTrace(null)); + StringBuilder builder = new StringBuilder(); + assertDoesNotThrow(() -> RingBufferEvent.printToStringArrayList(builder, "[", "]", 1, 2, 3, 4)); + assertDoesNotThrow(() -> RingBufferEvent.printToStringArrayList(builder, null, "]", 1, 2, 3, 4)); + assertDoesNotThrow(() -> RingBufferEvent.printToStringArrayList(builder, "[", null, 1, 2, 3, 4)); + assertDoesNotThrow(() -> RingBufferEvent.printToStringArrayList(builder, "", "]", 1, 2, 3, 4)); + assertDoesNotThrow(() -> RingBufferEvent.printToStringArrayList(builder, "[", "", 1, 2, 3, 4)); + } + + private static class BogusFilter implements Filter { + public BogusFilter() { + throw new IllegalStateException("should not call/use this filter"); + } + + @Override + public void clear() { + // never called + } + + @Override + public void copyTo(final Filter other) { + // never called + } + } +} diff --git a/core/src/test/java/io/opencmw/domain/BinaryDataTest.java b/core/src/test/java/io/opencmw/domain/BinaryDataTest.java new file mode 100644 index 00000000..7690b47e --- /dev/null +++ b/core/src/test/java/io/opencmw/domain/BinaryDataTest.java @@ -0,0 +1,110 @@ +package io.opencmw.domain; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.ThrowingSupplier; + +import io.opencmw.MimeType; + +class BinaryDataTest { + @Test + void testFixPreAndPost() { + Assertions.assertDoesNotThrow(() -> BinaryData.fixPreAndPost("test")); + assertEquals("/", BinaryData.fixPreAndPost(null)); + assertEquals("/", BinaryData.fixPreAndPost("")); + assertEquals("/", BinaryData.fixPreAndPost("/")); + assertEquals("/test/", BinaryData.fixPreAndPost("test")); + assertEquals("/test/", BinaryData.fixPreAndPost("/test")); + assertEquals("/test/", BinaryData.fixPreAndPost("test/")); + assertEquals("/test/", BinaryData.fixPreAndPost("/test/")); + } + + @Test + void testGenExportName() { + Assertions.assertDoesNotThrow(() -> BinaryData.genExportName("test")); + assertThrows(IllegalArgumentException.class, () -> BinaryData.genExportName(null)); + assertThrows(IllegalArgumentException.class, () -> BinaryData.genExportName("")); + assertEquals("test", BinaryData.genExportName("test.png")); + assertEquals("test", BinaryData.genExportName("test/test.png")); + assertEquals("test", BinaryData.genExportName("/test/test.png")); + assertEquals("test", BinaryData.genExportName("testA/testB/test.png")); + assertEquals("test", BinaryData.genExportName("testA/testB/test")); + + Assertions.assertDoesNotThrow(() -> BinaryData.genExportNameData("test.png")); + assertThrows(IllegalArgumentException.class, () -> BinaryData.genExportNameData(null)); + assertThrows(IllegalArgumentException.class, () -> BinaryData.genExportNameData("")); + assertEquals("test.png", BinaryData.genExportNameData("test.png")); + assertEquals("test.png", BinaryData.genExportNameData("test/test.png")); + assertEquals("test.png", BinaryData.genExportNameData("/test/test.png")); + assertEquals("test.png", BinaryData.genExportNameData("testA/testB/test.png")); + assertEquals("test", BinaryData.genExportNameData("testA/testB/test")); + } + + @Test + void testConstructor() { + assertDoesNotThrow((ThrowingSupplier) BinaryData::new); + assertDoesNotThrow(() -> new BinaryData("name2", MimeType.BINARY, new byte[0])); + assertThrows(IllegalArgumentException.class, () -> new BinaryData(null, MimeType.BINARY, new byte[0])); + assertThrows(IllegalArgumentException.class, () -> new BinaryData("name2", null, new byte[0])); + assertThrows(IllegalArgumentException.class, () -> new BinaryData("name2", MimeType.BINARY, null)); + assertThrows(IllegalArgumentException.class, () -> new BinaryData("name2", MimeType.BINARY, new byte[0], 1)); + } + + @Test + void testToString() { + final BinaryData testData = new BinaryData("name1", MimeType.TEXT, "testText".getBytes(StandardCharsets.UTF_8)); + assertNotNull(testData.toString(), "not null"); + // assert that sub-components are part of the description message + assertThat(testData.toString(), containsString(BinaryData.class.getSimpleName())); + assertThat(testData.toString(), containsString("name1")); + assertThat(testData.toString(), containsString("text/plain")); + assertThat(testData.toString(), containsString("testText")); + } + + @Test + void testEquals() { + final BinaryData reference = new BinaryData("name1", MimeType.TEXT, "testText".getBytes(StandardCharsets.UTF_8)); + final BinaryData test = new BinaryData("name2", MimeType.BINARY, "otherText".getBytes(StandardCharsets.UTF_8)); + assertThat("equality of identity", reference, is(equalTo(reference))); + assertThat("inequality for different object", reference, not(equalTo(new Object()))); + assertThat("inequality for resourceName differences", test, not(equalTo(reference))); + test.resourceName = reference.resourceName; + assertThat("inequality for MimeType differences", test, not(equalTo(reference))); + test.contentType = reference.contentType; + assertThat("inequality for data differences", test, not(equalTo(reference))); + test.data = reference.data; + assertThat("equality for content-effective copy", test, is(equalTo(reference))); + } + + @Test + void testHashCode() { + final BinaryData reference = new BinaryData("name1", MimeType.TEXT, "testText".getBytes(StandardCharsets.UTF_8)); + final BinaryData test = new BinaryData("name2", MimeType.BINARY, "otherText".getBytes(StandardCharsets.UTF_8)); + assertThat("uninitialised hashCode()", reference.hashCode(), is(not(equalTo(0)))); + assertThat("simple hashCode inequality", reference.hashCode(), is(not(equalTo(test.hashCode())))); + } + + @Test + void testGetCategory() { + Assertions.assertDoesNotThrow(() -> BinaryData.getCategory("test")); + assertThrows(IllegalArgumentException.class, () -> BinaryData.getCategory(null)); + assertThrows(IllegalArgumentException.class, () -> BinaryData.getCategory("")); + assertEquals("/", BinaryData.getCategory("test.png")); + assertEquals("/test/", BinaryData.getCategory("test/test.png")); + assertEquals("/test/", BinaryData.getCategory("/test/test.png")); + assertEquals("/testA/testB/", BinaryData.getCategory("testA/testB/test.png")); + assertEquals("/testA/testB/", BinaryData.getCategory("testA/testB/test")); + } +} \ No newline at end of file diff --git a/core/src/test/java/io/opencmw/filter/EvtTypeFilterTests.java b/core/src/test/java/io/opencmw/filter/EvtTypeFilterTests.java new file mode 100644 index 00000000..28c00f56 --- /dev/null +++ b/core/src/test/java/io/opencmw/filter/EvtTypeFilterTests.java @@ -0,0 +1,77 @@ +package io.opencmw.filter; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class EvtTypeFilterTests { + @Test + void basicTests() { + assertDoesNotThrow(EvtTypeFilter::new); + + final EvtTypeFilter evtTypeFilter = new EvtTypeFilter(); + assertInitialised(evtTypeFilter); + + evtTypeFilter.clear(); + assertInitialised(evtTypeFilter); + + assertNotNull(evtTypeFilter.toString()); + } + + @Test + void testEqualsAndHash() { + final EvtTypeFilter evtTypeFilter1 = new EvtTypeFilter(); + evtTypeFilter1.evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evtTypeFilter1.typeName = "DeviceName"; + // check identity + assertEquals(evtTypeFilter1, evtTypeFilter1); + assertEquals(evtTypeFilter1.hashCode(), evtTypeFilter1.hashCode()); + assertTrue(EvtTypeFilter.isDeviceData().test(evtTypeFilter1)); + assertTrue(EvtTypeFilter.isDeviceData("DeviceName").test(evtTypeFilter1)); + + assertNotEquals(evtTypeFilter1, new Object()); + + final EvtTypeFilter evtTypeFilter2 = new EvtTypeFilter(); + evtTypeFilter2.evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evtTypeFilter2.typeName = "DeviceName"; + assertEquals(evtTypeFilter1, evtTypeFilter2); + assertEquals(evtTypeFilter1.hashCode(), evtTypeFilter2.hashCode()); + + evtTypeFilter2.typeName = "DeviceName2"; + assertNotEquals(evtTypeFilter1, evtTypeFilter2); + evtTypeFilter2.evtType = EvtTypeFilter.DataType.PROCESSED_DATA; + + final EvtTypeFilter evtTypeFilter3 = new EvtTypeFilter(); + assertNotEquals(evtTypeFilter1, evtTypeFilter3); + assertDoesNotThrow(() -> evtTypeFilter1.copyTo(null)); + assertDoesNotThrow(() -> evtTypeFilter1.copyTo(evtTypeFilter3)); + assertEquals(evtTypeFilter1, evtTypeFilter3); + } + + @Test + void predicateTsts() { + final EvtTypeFilter evtTypeFilter = new EvtTypeFilter(); + + evtTypeFilter.evtType = EvtTypeFilter.DataType.TIMING_EVENT; + evtTypeFilter.typeName = "TimingEventName"; + assertTrue(EvtTypeFilter.isTimingData().test(evtTypeFilter)); + assertTrue(EvtTypeFilter.isTimingData("TimingEventName").test(evtTypeFilter)); + + evtTypeFilter.evtType = EvtTypeFilter.DataType.DEVICE_DATA; + evtTypeFilter.typeName = "DeviceName"; + assertTrue(EvtTypeFilter.isDeviceData().test(evtTypeFilter)); + assertTrue(EvtTypeFilter.isDeviceData("DeviceName").test(evtTypeFilter)); + + evtTypeFilter.evtType = EvtTypeFilter.DataType.SETTING_SUPPLY_DATA; + evtTypeFilter.typeName = "SettingName"; + assertTrue(EvtTypeFilter.isSettingsData().test(evtTypeFilter)); + assertTrue(EvtTypeFilter.isSettingsData("SettingName").test(evtTypeFilter)); + } + + private static void assertInitialised(final EvtTypeFilter evtTypeFilter) { + assertNotNull(evtTypeFilter.typeName); + assertTrue(evtTypeFilter.typeName.isBlank()); + assertEquals(EvtTypeFilter.DataType.UNKNOWN, evtTypeFilter.evtType); + assertEquals(0, evtTypeFilter.hashCode); + } +} diff --git a/core/src/test/java/io/opencmw/filter/TimingCtxTests.java b/core/src/test/java/io/opencmw/filter/TimingCtxTests.java new file mode 100644 index 00000000..42bc810b --- /dev/null +++ b/core/src/test/java/io/opencmw/filter/TimingCtxTests.java @@ -0,0 +1,163 @@ +package io.opencmw.filter; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class TimingCtxTests { + @Test + void basicTests() { + assertDoesNotThrow(TimingCtx::new); + + final long timeNowMicros = System.currentTimeMillis() * 1000; + final TimingCtx ctx = new TimingCtx(); + assertInitialised(ctx); + + assertDoesNotThrow(() -> ctx.setSelector("FAIR.SELECTOR.C=0:S=1:P=2:T=3", timeNowMicros)); + assertEquals(0, ctx.cid); + assertEquals(1, ctx.sid); + assertEquals(2, ctx.pid); + assertEquals(3, ctx.gid); + assertEquals(timeNowMicros, ctx.bpcts); + assertNotNull(ctx.toString()); + ctx.clear(); + + assertThrows(IllegalArgumentException.class, () -> ctx.setSelector("FAIR.SELECTOR.C=0:S=1:P=2:T=3", -1)); + assertInitialised(ctx); + + // add unknown/erroneous tag + assertThrows(IllegalArgumentException.class, () -> ctx.setSelector("FAIR.SELECTOR.C0:S=1:P=2:T=3", timeNowMicros)); + assertInitialised(ctx); + assertThrows(IllegalArgumentException.class, () -> ctx.setSelector("FAIR.SELECTOR.X=1", timeNowMicros)); + assertInitialised(ctx); + + assertThrows(IllegalArgumentException.class, () -> ctx.setSelector(null, timeNowMicros)); + assertInitialised(ctx); + } + + @Test + void basicAllSelectorTests() { + final long timeNowMicros = System.currentTimeMillis() * 1000; + final TimingCtx ctx = new TimingCtx(); + + // empty selector + assertDoesNotThrow(() -> ctx.setSelector("", timeNowMicros)); + assertEquals("", ctx.selector.toUpperCase()); + assertAllWildCard(ctx); + assertEquals(timeNowMicros, ctx.bpcts); + + // "ALL" selector + assertDoesNotThrow(() -> ctx.setSelector(TimingCtx.WILD_CARD, timeNowMicros)); + assertEquals(TimingCtx.WILD_CARD, ctx.selector.toUpperCase()); + assertAllWildCard(ctx); + assertEquals(timeNowMicros, ctx.bpcts); + + // "FAIR.SELECTOR.ALL" selector + assertDoesNotThrow(() -> ctx.setSelector("FAIR.SELECTOR.ALL", timeNowMicros)); + assertEquals(TimingCtx.SELECTOR_PREFIX + TimingCtx.WILD_CARD, ctx.selector.toUpperCase()); + assertAllWildCard(ctx); + assertEquals(timeNowMicros, ctx.bpcts); + } + + @Test + void testHelper() { + assertTrue(TimingCtx.wildCardMatch(TimingCtx.WILD_CARD_VALUE, 2)); + assertTrue(TimingCtx.wildCardMatch(TimingCtx.WILD_CARD_VALUE, -1)); + assertTrue(TimingCtx.wildCardMatch(1, TimingCtx.WILD_CARD_VALUE)); + assertTrue(TimingCtx.wildCardMatch(-1, TimingCtx.WILD_CARD_VALUE)); + assertFalse(TimingCtx.wildCardMatch(3, 2)); + } + + @Test + void testEqualsAndHash() { + final long timeNowMicros = System.currentTimeMillis() * 1000; + final TimingCtx ctx1 = new TimingCtx(); + assertDoesNotThrow(() -> ctx1.setSelector("FAIR.SELECTOR.C=0:S=1:P=2:T=3", timeNowMicros)); + // check identity + assertEquals(ctx1, ctx1); + assertEquals(ctx1.hashCode(), ctx1.hashCode()); + + assertNotEquals(ctx1, new Object()); + + final TimingCtx ctx2 = new TimingCtx(); + assertDoesNotThrow(() -> ctx2.setSelector("FAIR.SELECTOR.C=0:S=1:P=2:T=3", timeNowMicros)); + + assertEquals(ctx1, ctx2); + assertEquals(ctx1.hashCode(), ctx2.hashCode()); + + ctx2.bpcts++; + assertNotEquals(ctx1, ctx2); + ctx2.gid = -1; + assertNotEquals(ctx1, ctx2); + ctx2.pid = -1; + assertNotEquals(ctx1, ctx2); + ctx2.sid = -1; + assertNotEquals(ctx1, ctx2); + ctx2.cid = -1; + assertNotEquals(ctx1, ctx2); + + final TimingCtx ctx3 = new TimingCtx(); + assertNotEquals(ctx1, ctx3); + assertDoesNotThrow(() -> ctx1.copyTo(null)); + assertDoesNotThrow(() -> ctx1.copyTo(ctx3)); + assertEquals(ctx1, ctx3); + } + + @Test + void basicSelectorTests() { + final long timeNowMicros = System.currentTimeMillis() * 1000; + final TimingCtx ctx = new TimingCtx(); + assertInitialised(ctx); + + // "FAIR.SELECTOR.C=2" selector + assertDoesNotThrow(() -> ctx.setSelector("FAIR.SELECTOR.C=2", timeNowMicros)); + assertEquals(2, ctx.cid); + assertEquals(-1, ctx.sid); + assertEquals(-1, ctx.pid); + assertEquals(-1, ctx.gid); + assertEquals(timeNowMicros, ctx.bpcts); + } + + @Test + void matchingTests() { // NOPMD NOSONAR -- number of assertions is OK ... it's a simple unit-test + final long timeNowMicros = System.currentTimeMillis() * 1000; + final TimingCtx ctx = new TimingCtx(); + ctx.setSelector("FAIR.SELECTOR.C=0:S=1:P=2:T=3", timeNowMicros); + + assertTrue(ctx.matches(ctx).test(ctx)); + assertTrue(TimingCtx.matches(0, timeNowMicros).test(ctx)); + assertTrue(TimingCtx.matches(0, 1, timeNowMicros).test(ctx)); + assertTrue(TimingCtx.matches(0, 1, 2, timeNowMicros).test(ctx)); + assertTrue(TimingCtx.matches(0, 1, 2).test(ctx)); + assertTrue(TimingCtx.matches(-1, 1, 2).test(ctx)); + assertFalse(TimingCtx.matches(0, 0, 2).test(ctx)); + assertFalse(TimingCtx.matches(0, 1, 0).test(ctx)); + + assertTrue(TimingCtx.matchesBpcts(timeNowMicros).test(ctx)); + assertTrue(TimingCtx.isOlderBpcts(timeNowMicros + 1L).test(ctx)); + assertTrue(TimingCtx.isNewerBpcts(timeNowMicros - 1L).test(ctx)); + + // test wildcard + ctx.setSelector("FAIR.SELECTOR.C=0:S=1", timeNowMicros); + assertEquals(0, ctx.cid); + assertEquals(1, ctx.sid); + assertEquals(-1, ctx.pid); + assertTrue(TimingCtx.matches(0, 1).test(ctx)); + assertTrue(TimingCtx.matches(0, 1, -1).test(ctx)); + assertTrue(TimingCtx.matches(0, timeNowMicros).test(ctx)); + } + + private static void assertAllWildCard(final TimingCtx ctx) { + assertEquals(-1, ctx.cid); + assertEquals(-1, ctx.sid); + assertEquals(-1, ctx.pid); + assertEquals(-1, ctx.gid); + } + + private static void assertInitialised(final TimingCtx ctx) { + assertEquals("", ctx.selector); + assertAllWildCard(ctx); + assertEquals(-1, ctx.bpcts); + assertEquals(0, ctx.hashCode); + } +} diff --git a/core/src/test/java/io/opencmw/rbac/BasicRbacRoleTest.java b/core/src/test/java/io/opencmw/rbac/BasicRbacRoleTest.java new file mode 100644 index 00000000..502fc694 --- /dev/null +++ b/core/src/test/java/io/opencmw/rbac/BasicRbacRoleTest.java @@ -0,0 +1,9 @@ +package io.opencmw.rbac; + +class BasicRbacRoleTest { + // @Test + // void testBasicRbac() { + // assertTrue(BasicRbacRole.NULL.compareTo(BasicRbacRole.ADMIN) > 0, "ADMIN higher than NULL"); + // assertTrue(BasicRbacRole.ADMIN.compareTo(BasicRbacRole.NULL) < 0, "ADMIN higher than NULL"); + // } +} \ No newline at end of file diff --git a/core/src/test/java/io/opencmw/utils/CacheTests.java b/core/src/test/java/io/opencmw/utils/CacheTests.java new file mode 100644 index 00000000..4c3c1512 --- /dev/null +++ b/core/src/test/java/io/opencmw/utils/CacheTests.java @@ -0,0 +1,226 @@ +package io.opencmw.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Regression testing for @see Cache + * + * @author rstein + */ +@Execution(ExecutionMode.SAME_THREAD) +public class CacheTests { + private static final Logger LOGGER = LoggerFactory.getLogger(CacheTests.class); + + @Test + public void demoTestCase() { + AtomicBoolean preListenerCalled = new AtomicBoolean(false); + AtomicBoolean postListenerCalled = new AtomicBoolean(false); + final Cache cache = Cache.builder().withLimit(10).withTimeout(100, TimeUnit.MILLISECONDS) // + .withPreListener((k, v) -> preListenerCalled.set(true)) + .withPostListener((k, v) -> postListenerCalled.set(true)) + .build(); + + String name1 = "Han Solo"; + + cache.put(name1, 10); + assertTrue(isCached(cache, name1), "initial push"); + + // wait 1 second + try { + Thread.sleep(50); + } catch (InterruptedException e) { + LOGGER.atError().setCause(e).log("sleep"); + } + + assertTrue(isCached(cache, name1), "check after 500 ms"); + assertFalse(preListenerCalled.get()); + assertFalse(postListenerCalled.get()); + + // wait another second + try { + Thread.sleep(200); + } catch (InterruptedException e) { + LOGGER.atError().setCause(e).log("sleep"); + } + assertFalse(isCached(cache, name1), "check after 500 ms"); + assertTrue(preListenerCalled.get()); + assertTrue(postListenerCalled.get()); + } + + @Test + public void testCacheSizeLimit() { + Cache cache = Cache.builder().withLimit(3).build(); + + assertEquals(3, cache.getLimit()); + + for (int i = 0; i < 10; i++) { + cache.put("test" + i, 10); + if (i < cache.getLimit()) { + assertEquals(i + 1, cache.getSize()); + } + assertTrue(cache.getSize() <= 3, "cache size during iteration " + i); + } + assertEquals(3, cache.getLimit()); + + final String testString = "testString"; + cache.put(testString, 42); + assertTrue(isCached(cache, testString), testString + " being cached"); + assertEquals(42, cache.get(testString), testString + " being cached"); + cache.remove(testString); + assertFalse(isCached(cache, testString), testString + " being removed from cache"); + + cache.clear(); + assertEquals(0, cache.size(), "cache size"); + cache.put(testString, 42); + assertTrue(cache.containsKey(testString), "containsKey"); + assertTrue(cache.containsValue(42), "containsValue"); + Set> entrySet = cache.entrySet(); + + assertEquals(1, entrySet.size(), "entrySet size"); + for (Entry entry : entrySet) { + assertEquals(testString, entry.getKey(), "entrySet - key"); + assertEquals(42, entry.getValue(), "entrySet - value"); + } + + Set keySet = cache.keySet(); + assertEquals(1, keySet.size(), "keySet size"); + for (String key : keySet) { + assertEquals(testString, key, "keySet - key"); + } + + Collection values = cache.values(); + assertEquals(1, values.size(), "values size"); + for (Integer value : values) { + assertEquals(42, value, "values - value"); + } + + assertEquals(1, cache.size(), "cache size"); + cache.clear(); + assertEquals(0, cache.size(), "cache size"); + assertFalse(isCached(cache, testString), testString + " being removed from cache"); + assertTrue(cache.isEmpty(), " cache being empty after clear"); + + Map mapToAdd = new ConcurrentHashMap<>(); + mapToAdd.put("Test1", 1); + mapToAdd.put("Test2", 2); + mapToAdd.put("Test3", 3); + cache.putAll(mapToAdd); + assertEquals(3, cache.size(), "cache size"); + } + + @Test + public void testConstructors() { + Cache cache1 = new Cache<>(20); // limit + assertEquals(20, cache1.getLimit(), "limit"); + assertDoesNotThrow(() -> cache1.put("testKey", "testValue")); + Cache cache2 = new Cache<>(1000, TimeUnit.MILLISECONDS); // time-out + assertEquals(1000, cache2.getTimeout(), "time out"); + assertEquals(TimeUnit.MILLISECONDS, cache2.getTimeUnit(), "time unit"); + assertDoesNotThrow(() -> cache2.put("testKey", "testValue")); + Cache cache3 = new Cache<>(1000, TimeUnit.MILLISECONDS, 20); // time-out && limit + assertEquals(20, cache3.getLimit(), "limit"); + assertEquals(TimeUnit.MILLISECONDS, cache3.getTimeUnit(), "limit"); + assertEquals(1000, cache3.getTimeout(), "limit"); + assertDoesNotThrow(() -> cache3.put("testKey", "testValue")); + + // check exceptions + + assertThrows(IllegalArgumentException.class, () -> { + // negative time out check + new Cache(-1, TimeUnit.MILLISECONDS, 20); + }); + + assertThrows(IllegalArgumentException.class, () -> { + // null TimeUnit check + new Cache(1, null, 20); + }); + + assertThrows(IllegalArgumentException.class, () -> { + // limit < 1 check + new Cache(2, TimeUnit.MICROSECONDS, 0); + }); + + // check builder exceptions + + assertThrows(IllegalArgumentException.class, () -> { + // negative time out check + Cache.builder().withTimeout(-1, TimeUnit.MILLISECONDS).build(); + }); + + assertThrows(IllegalArgumentException.class, () -> { + // null TimeUnit check + Cache.builder().withTimeout(1, null).build(); + }); + + assertThrows(IllegalArgumentException.class, () -> { + // limit < 1 check + Cache.builder().withLimit(0).build(); + }); + + assertThrows(IllegalArgumentException.class, () -> { + // null pre-listener + Cache.builder().withPreListener(null).build(); + }); + + assertThrows(IllegalArgumentException.class, () -> { + // null post-listener + Cache.builder().withPostListener(null).build(); + }); + + // Cache cache4 = Cache.builder().withLimit(20).withTimeout(100, TimeUnit.MILLISECONDS).build(); + } + + @Test + public void testHelperMethods() { + // TimeUnit to ChronoUnit conversions + for (TimeUnit timeUnit : TimeUnit.values()) { + ChronoUnit chronoUnit = Cache.convertToChronoUnit(timeUnit); + // timeUnit.toChronoUnit() would be faster but exists only since Java 9 + + long nanoTimeUnit = timeUnit.toNanos(1); + long nanoChrono = chronoUnit.getDuration().getNano() + 1000000000 * chronoUnit.getDuration().getSeconds(); + assertEquals(nanoTimeUnit, nanoChrono, "ChronoUnit =" + chronoUnit); + } + + // test clamp(int ... ) routine + assertEquals(1, Cache.clamp(1, 3, 0)); + assertEquals(2, Cache.clamp(1, 3, 2)); + assertEquals(3, Cache.clamp(1, 3, 4)); + + // test clamp(long ... ) routine + assertEquals(1L, Cache.clamp(1L, 3L, 0L)); + assertEquals(2L, Cache.clamp(1L, 3L, 2L)); + assertEquals(3L, Cache.clamp(1L, 3L, 4L)); + } + + @Test + public void testPutVariants() { + Cache cache = Cache.builder().withLimit(3).build(); + + assertNull(cache.put("key", 2)); + assertEquals(2, cache.put("key", 3)); + assertEquals(3, cache.putIfAbsent("key", 4)); + cache.clear(); + assertNull(cache.putIfAbsent("key", 4)); + assertEquals(4, cache.putIfAbsent("key", 5)); + } + + private static boolean isCached(Cache cache, final String key) { + return cache.getOptional(key).isPresent(); + } +} diff --git a/core/src/test/java/io/opencmw/utils/CustomFutureTests.java b/core/src/test/java/io/opencmw/utils/CustomFutureTests.java new file mode 100644 index 00000000..59e832df --- /dev/null +++ b/core/src/test/java/io/opencmw/utils/CustomFutureTests.java @@ -0,0 +1,157 @@ +package io.opencmw.utils; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.*; + +import java.net.ProtocolException; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +import org.junit.jupiter.api.Test; + +class CustomFutureTests { + @Test + void testWithoutWaiting() throws ExecutionException, InterruptedException { + final CustomFuture future = new CustomFuture<>(); + + assertFalse(future.done.get(), "future is active"); + assertFalse(future.isCancelled()); + future.setReply("TestString"); + + assertEquals("TestString", future.get()); + assertFalse(future.isCancelled()); + } + + @Test + void testWithWaiting() { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + assertFalse(future.isCancelled()); + + final AtomicReference result = new AtomicReference<>(); + final AtomicBoolean run = new AtomicBoolean(false); + new Thread(() -> { + run.set(true); + try { + result.set(future.get()); + assertEquals("TestString", future.get()); + assertEquals("TestString", result.get()); + } catch (InterruptedException | ExecutionException e) { + throw new IllegalStateException("unexpected exception", e); + } + run.set(false); + }).start(); + await().alias("wait for thread to start").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(true)); + future.setReply("TestString"); + await().alias("wait for thread to finish").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(false)); + + assertEquals("TestString", result.get()); + assertFalse(future.isCancelled()); + } + + @Test + void testWithExecutionException() { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + assertFalse(future.isCancelled()); + future.setException(new ProtocolException("specific exception")); + + assertThrows(ExecutionException.class, future::get); + + assertThrows(IllegalStateException.class, () -> future.setException(new ProtocolException("specific exception"))); + } + + @Test + void testWithExecutionExceptionWhileWaiting() { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + + new Thread(() -> { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); + future.setException(new ProtocolException("specific exception")); + }).start(); + assertThrows(ExecutionException.class, () -> future.get(1000, TimeUnit.MILLISECONDS)); + } + + @Test + void testWithCancelWhileWaiting() { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + assertFalse(future.isCancelled()); + + final AtomicReference result = new AtomicReference<>(); + final AtomicBoolean run = new AtomicBoolean(false); + new Thread(() -> { + run.set(true); + assertThrows(CancellationException.class, () -> result.set(future.get())); + run.set(false); + }).start(); + await().alias("wait for thread to start").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(true)); + assertFalse(future.isCancelled()); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(200)); + assertTrue(future.cancel(true)); + assertFalse(future.cancel(true)); + assertTrue(future.isCancelled()); + await().alias("wait for thread to finish").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(false)); + assertThrows(IllegalStateException.class, () -> future.setReply("TestString")); + + assertNull(result.get()); + } + + @Test + void testWithTimeout() { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + assertThrows(TimeoutException.class, () -> future.get(100, TimeUnit.MILLISECONDS)); + } + + @Test + void testWithTimeoutAndCancel() { + final CustomFuture future = new CustomFuture<>(); + final AtomicBoolean run = new AtomicBoolean(false); + final Thread testThread = new Thread(() -> { + run.set(true); + assertThrows(CancellationException.class, () -> future.get(1, TimeUnit.SECONDS)); + run.set(false); + }); + testThread.start(); + await().alias("wait for thread to start").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(true)); + future.cancel(false); + await().alias("wait for thread to finish").atMost(10, TimeUnit.SECONDS).until(run::get, equalTo(false)); + } + + @Test + void testWithCancelBeforeWaiting() { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + assertFalse(future.isCancelled()); + assertTrue(future.cancel(true)); + assertFalse(future.cancel(true)); + + final AtomicReference result = new AtomicReference<>(); + final AtomicBoolean run = new AtomicBoolean(true); + new Thread(() -> { + assertThrows(CancellationException.class, () -> result.set(future.get())); + run.set(false); + }).start(); + assertTrue(future.isCancelled()); + assertThrows(IllegalStateException.class, () -> future.setReply("TestString")); + await().alias("wait for thread to finish").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(false)); + + assertNull(result.get()); + } + + @Test + void testWithNullReply() throws ExecutionException, InterruptedException { + final CustomFuture future = new CustomFuture<>(); + assertFalse(future.done.get(), "future is active"); + future.setReply(null); + assertNull(future.get()); + } +} diff --git a/core/src/test/java/io/opencmw/utils/LimitedArrayListTests.java b/core/src/test/java/io/opencmw/utils/LimitedArrayListTests.java new file mode 100644 index 00000000..1e02512d --- /dev/null +++ b/core/src/test/java/io/opencmw/utils/LimitedArrayListTests.java @@ -0,0 +1,48 @@ +package io.opencmw.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit testing for {@link LimitedArrayListTests} implementation. + * + * @author rstein + */ +class LimitedArrayListTests { + @Test + void testConstructors() { + Assertions.assertDoesNotThrow(() -> new LimitedArrayList<>(10)); + + assertThrows(IllegalArgumentException.class, () -> new LimitedArrayList<>(0)); + assertThrows(IllegalArgumentException.class, () -> new LimitedArrayList<>(-1)); + + LimitedArrayList testList = new LimitedArrayList<>(1); + assertEquals(1, testList.getLimit()); + assertEquals(0, testList.size()); + + assertThrows(IllegalArgumentException.class, () -> testList.setLimit(0)); + testList.setLimit(3); + assertEquals(3, testList.getLimit()); + + assertEquals(0, testList.size()); + assertTrue(testList.add(1.0)); + assertEquals(1, testList.size()); + assertTrue(testList.add(2.0)); + assertEquals(2, testList.size()); + assertTrue(testList.add(3.0)); + assertEquals(3, testList.size()); + assertTrue(testList.add(4.0)); + assertEquals(3, testList.size()); + + testList.clear(); + for (int i = 0; i <= 13; i++) { + testList.add((double) i); + } + assertEquals(3, testList.size()); + assertEquals(11.0, testList.get(0)); + assertEquals(12.0, testList.get(1)); + assertEquals(13.0, testList.get(2)); + } +} diff --git a/core/src/test/java/io/opencmw/utils/SharedPointerTests.java b/core/src/test/java/io/opencmw/utils/SharedPointerTests.java new file mode 100644 index 00000000..9a3b975d --- /dev/null +++ b/core/src/test/java/io/opencmw/utils/SharedPointerTests.java @@ -0,0 +1,41 @@ +package io.opencmw.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.ThrowingSupplier; + +class SharedPointerTests { + @Test + void basicTests() { + assertDoesNotThrow((ThrowingSupplier>) SharedPointer::new); + + final SharedPointer sp = new SharedPointer<>(); + final Integer testObject = 2; + AtomicInteger destroyed = new AtomicInteger(0); + // set and get first ownership + sp.set(testObject, obj -> { + // destroyed called + assertEquals(obj, testObject, "lambda object equality"); + destroyed.getAndIncrement(); + }); + assertThrows(IllegalStateException.class, () -> sp.set(testObject)); + assertEquals(Integer.class, sp.getType()); + assertEquals(1, sp.getReferenceCount()); + + // get second ownership + final SharedPointer ref = sp.getCopy(); + assertEquals(testObject, ref.get(), "object identity copy"); + assertEquals(testObject, ref.get(Integer.class), "object identity copy"); + assertThrows(ClassCastException.class, () -> ref.get(Long.class)); + assertEquals(2, sp.getReferenceCount()); + assertDoesNotThrow(ref::release); // nothing should happen + assertEquals(1, ref.getReferenceCount()); + assertEquals(0, destroyed.get(), "erroneous destructor call"); + + assertDoesNotThrow(sp::release); // nothing should happen + assertEquals(1, destroyed.get(), "destructor not called"); + } +} diff --git a/core/src/test/resources/simplelogger.properties b/core/src/test/resources/simplelogger.properties new file mode 100644 index 00000000..a01ef764 --- /dev/null +++ b/core/src/test/resources/simplelogger.properties @@ -0,0 +1,50 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=debug + +# The output target which can be the path to a file, or the special values "System.out" and "System.err". +# Default is "System.err". +org.slf4j.simpleLogger.logFile=System.out + +# If the output target is set to "System.out" or "System.err" (see preceding entry), by default, +# logs will be output to the latest value referenced by System.out/err variables. +# By setting this parameter to true, the output stream will be cached, i.e. assigned once at initialization +# time and re-used independently of the current value referenced by System.out/err. +org.slf4j.simpleLogger.cacheOutputStream=true + +# Logging detail level for a SimpleLogger instance named "a.b.c". Right-side value must be one of +# "trace", "debug", "info", "warn", "error" or "off". When a SimpleLogger named "a.b.c" is initialized, +# its level is assigned from this property. If unspecified, the level of nearest parent logger will be used, +# and if none is set, then the value specified by org.slf4j.simpleLogger.defaultLogLevel will be used. +org.slf4j.simpleLogger.log.de.gsi.* + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +# org.slf4j.simpleLogger.showThreadName=false + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +# org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/docs/CmwLight.md b/docs/CmwLight.md new file mode 100644 index 00000000..f75f0c29 --- /dev/null +++ b/docs/CmwLight.md @@ -0,0 +1,172 @@ +# Unofficial CMW client protocol implementation + +This is an effort of documenting and reimplementing the CMW (Controls Middleware) +binary protocol. + +CMW is a project developed at CERN, for more information see its +[online documentation](https://cmwdoc.web.cern.ch/cmwdoc/) or the [paper](https://cds.cern.ch/record/2305650/files/mobpl05.pdf). + +## CMW protocol description + +The CMW rda3 protocol consists of individual [ZeroMQ](https://zeromq.org/) messages with +one to four frames. +The frames either contain simple enum-type bytes, a byte encoded string or CMW data encoded binary data. +The CMW data format is reimplemented in the `CmwLightSerialiser`, see the [serialiser documentation](IoSerialiser.md) +for more information. +The following sections describe the byte values and cmw-data field names used by the protocol +and their use for establishing a connection, performing subscribe/get/... actions and receiving the replies. + +The first ZFrame of each message contains a single byte defining the type of the Message: + +| byte value | message name | direction | contents | +|------------|--------------------|-----------|----------| +| 0x01 | SERVER_CONNECT_ACK | s -> c | MessageType, VersionString +| 0x02 | SERVER_REP | s -> c | MessageType, Frames(1-3), Descriptor +| 0x03 | SERVER_HB | s -> c | MessageType +| 0x20 | CLIENT_CONNECT | c -> s | MessageType, VersionString +| 0x21 | CLIENT_REQ | c -> s | MessageType, Frames(1-3), Descriptor +| 0x22 | CLIENT_HB | c -> s | MessageType + +### Establishing the connection + +The connection is established by the client sending a ZMsg with the first frame containing +the message type CLIENT_CONNECT, followed a ZFrame containing the version string +"1.0.0" (the version string is unused). +The server acknowledges by sending SERVER_CONNECT_ACK, followed by the version +string ("1.0.0", not used). + +### Heartbeats + +The rda3 protocol uses heartbeats for connection management. +Client as well as server periodically send messages only consisting of a single one-byte frame containing SERVER_HB/ CLIENT_HB. +If client or server do not receive a heartbeat or any other message for some time, the connection is reset. +By default, both sides send a heartbeat every second and reset the connection if 3 consecutive heartbeats are missed. +Heartbeats are only sent if there is otherwise no communication, so every received package must also reset this timeout. + +### Requests/Replies + +The client can send requests and the server sends replies, indicated by the types CLIENT_REQ and SERVER_REP. +The message type frame is followed by an arbitrary number of frames, where the last one is the so called descriptor, +which contains one byte for each previous frame, containing the type of the frame contents. + +| message name | byte value | +|-------------------------|------------| +| MT_HEADER | 0 | +| MT_BODY | 1 | +| MT_BODY_DATA_CONTEXT | 2 | +| MT_BODY_REQUEST_CONTEXT | 3 | +| MT_BODY_EXCEPTION | 4 | + +The second frame is always of type MT_HEADER and its field reqType defines the type of the +request/reply and also the frames present in the data: + +| byte | message type | message | direction | comment | +|------|-----------------------|---------|-----------|---------| +| 0 |RT_GET | H, RC | C->S | | +| 1 |RT_SET | H,B,RC | C->S | | +| 2 |RT_CONNECT | H, B | C->S | | +| 3 |RT_REPLY | H,B | S->C | response to connect | +| 3 |RT_REPLY | H,B,DC | S->C | | +| 4 |RT_EXCEPTION | H,B | S->C | | +| 5 |RT_SUBSCRIBE | H, RC | C->S | | +| 5 |RT_SUBSCRIBE | H | S->C | "ack" | +| 6 |RT_UNSUBSCRIBE | H, RC | C->S | | +| 7 |RT_NOTIFICATION_DATA | H,B,DC | S->C | | +| 8 |RT_NOTIFICATION_EXC | H,B | S->C | | +| 9 |RT_SUBSCRIBE_EXCEPTION | H,B | S->C | | +| 10 |RT_EVENT | H | C->S | | +| 10 |RT_EVENT | H | S->C | close | +| 11 |RT_SESSION_CONFIRM | H | S->C | | + +#### Header fields: +The header frame is sent as the second frame of each message, but depending on the message and request type, +not all fields are populated and different option fields are present. + +| fieldname | enum tag | type | description | +|-----------|-------------------|------|-------------| +| "2" | REQ_TYPE_TAG | byte | see table above +| "0" | ID_TAG | long | id to map back to the request from the reply, for subscription replies set by source id from subscription reply +| "1" | DEVICE_NAME_TAG |string| empty for subscription notifications +| "7" | UPDATE_TYPE_TAG | byte | +| "d" | SESSION_ID_TAG |string| empty for subscription notifications +| "f" | PROPERTY_NAME_TAG |string| empty for subscription notifications +| "3" | OPTIONS_TAG | data | + +The Session Id tag contains the session id which identifies the client: + - `RemoteHostInfoImpl[name=; userName=; appId=[app=;uid=;host=;pid=;]` + +The options tag can contain the following fields + - optional NOTIFICATION_ID_TAG = "a" type: long; for notification data, counts the notifications + - optional SOURCE_ID_TAG = "b"; for subscription requests to propagate the id + - optional SESSION_BODY_TAG = "e" type: cmw-data; for session context/RBAC + +#### Request Context Fields +The request context frame is sent for get, set and subscribe requests and specifies on which data the request should act. + +| fieldname | enum tag | type | description | +|-----------|--------------|------|-------------| +| "8" | SELECTOR_TAG |string| | +| "c" | FILTERS_TAG | data | | +| "x" | DATA_TAG | data | | + +#### Data Context Fields +The data context frame is sent for reply and notification replies and specifies, what context the data belongs to. + +| fieldname | enum tag | type | description | +|-----------|-----------------|------|-------------| +| "4" | CYCLE_NAME_TAG |string| | +| "5" | ACQ_STAMP_TAG | data | | +| "6" | CYCLE_STAMP_TAG | data | | +| "x" | DATA_TAG | data | fields: "acqStamp", "cycleStamp", "cycleName", "version", "type"| + +#### connect body +The connection step seems to be optional, it will be established implicitly when opening a subscription. +The server responds with an empty (fields as well as data frame) `REPLY` message. + +| fieldname | enum tag | type | description | +|-----------|-----------------|------|-------------| +| "9" | CLIENT_INFO_TAG |string| | + +The field contains a string of `#` separated `key#type[#length]#value` entries. Strings are URL encoded. +- `Address:#string#16#tcp:%2F%2FSYSPC004:0` +- `ApplicationId:#string#54#app=fesa%2Dexplorer2;uid=akrimm;host=SYSPC004;pid=17442;` +- `UserName:#string#6#akrimm` +- `ProcessName:#string#14#fesa%2Dexplorer2` +- `Language:#string#4#Java` +- `StartTime:#long#1605172397732` +- `Name:#string#14#fesa%2Dexplorer2` +- `Pid:#int#17442` +- `Version:#string#5#2%2E8%2E1` + +#### Exception body field +If there is an exception, the server sends a reply of type exception, subscribe exception or notification exception. +The enclosed exception body frame contains the following fields. + +| fieldname | enum tag | type | description | +|---------------------|------------------------------------|------|-------------| +| "Message" |EXCEPTION_MESSAGE_FIELD |string| | +| "Type" |EXCEPTION_TYPE_FIELD |string| | +| "Backtrace" |EXCEPTION_BACKTRACE_FIELD |string| | +| "ContextCycleName" |EXCEPTION_CONTEXT_CYCLE_NAME_FIELD |string| | +| "ContextCycleStamp" |EXCEPTION_CONTEXT_CYCLE_STAMP_FIELD | long | | +| "ContextAcqStamp" |EXCEPTION_CONTEXT_ACQ_STAMP_FIELD | long | | +| "ContextData" |EXCEPTION_CONTEXT_DATA_FIELD |string| | + +#### currently unused field names +- MESSAGE_TAG = "message"; + + +## Client implementation + +The implementation only cares about the client part and for now only supports property subscriptions. +RBAC is also not supported. + +The `CmwLightMessage` implements a generic message which can represent all the messages exchanged between client and server. +It provides methods for generating consistent messages and static instances for the heartbeat messages. +The `CmwLightProtocol` takes care of translating these messages to and from ZeroMQ's `ZMsg` format. + +One `CmwLightClient` takes care of one cmw server. It manages connection state, subscriptions and their state. +It is supposed to be embedded into an event loop, which should call the receiveMessage call at least once every `heartbeatInterval`. +This can be facilitated efficiently by registering a ZeroMQ poller to the client's socket. + +See `CmwLightPoller` for an example which publishes the subscription notifications into an LMAX disruptor ring buffer. \ No newline at end of file diff --git a/docs/IoSerialiser.md b/docs/IoSerialiser.md new file mode 100644 index 00000000..26264b50 --- /dev/null +++ b/docs/IoSerialiser.md @@ -0,0 +1,303 @@ +# Yet-another-Serialiser (YaS) +### or: why we are not reusing ... and opted to write yet another custom data serialiser + +Data serialisation is a basic core functionality when information has to be transmitted, stored and later retrieved by (often quite) +different sub-systems. With a multitude of different serialiser libraries, a non-negligible subset of these claim to be the fastest, +most efficient, easiest-to-use or *<add your favourite superlative here>*. +While this may be certainly true for the libraries' original design use-case, this often breaks down for other applications +that are often quite diverse and may focus on different aspects depending on the application. Hence, a fair comparison of +their performance is usually rather complex and highly non-trivial because the underlying assumptions of 'what counts as important' +being quite different between specific domains, boundary constraints, application goals and resulting (de-)serialisation strategies. +Rather than claiming any superlative, or needlessly bashing solutions that are well suited for their design use-cases, we wanted +to document the considerations, constraints and application goals of our specific use-case and that guided our +[multi-protocol serialiser developments](https://github.com/GSI-CS-CO/chart-fx/microservice). +This also in the hope that it might find interest, perhaps adoption, inspires new ideas, or any other form of improvements. +Thus, if you find something missing, unclear, or things that could be improved, please feel encouraged to post a PR. + +DISCLAIMER: This specific implementation while not necessarily a direct one-to-one source-code copy is at least conceptually +based upon a combination of other open-sourced implementations, long-term experience with internal-proprietary wire-formats, +and new serialiser design ideas expressed in the references [below](#references) which were adopted, adapted and optimised +for our specific use-case. + +### [Our](https://fair-center.eu/) [Use-Case](https://fair-wiki.gsi.de/FC2WG) +We use [this](../../microservice) and [Chart-Fx](https://github.com/GSI-CS-CO/chart-fx) in order to aid the development +of functional microservices that monitor and control a large variety of device- and beam-based parameters that are necessary +for the operation of our [FAIR particle accelerators](https://www.youtube.com/watch?v=zy4b0ZQnsck). +These microservices cover in particular those that require the aggregation of measurement data from different sub-systems, +or that require domain-specific logic or real-time signal-processing algorithms that cannot be efficiently implemented +in any other single device or sub-system. + +### Quick Overview +This serialiser implementation defines three levels of interface abstractions: + * [IoBuffer](../../microservice/src/main/java/de/gsi/serializer/IoBuffer.java) which defines the low-level byte-array format + of how data primitives (ie. `boolean`, `byte`, ...,`float`, `double`), `String`, and their array counter-part (ie. + `boolean[]`, `byte[]`, ...,`float[]', 'double[]`, `String[]`) are stored. There are two default implementations: + - [ByteBuffer](../../microservice/src/main/java/de/gsi/serializer/spi/ByteBuffer.java) which basically wraps around and + extends `java.nio.ByteBuffer` to also support `String` and primitive arrays, and + - [FastByteBuffer](../../microservice/src/main/java/de/gsi/serializer/spi/FastByteBuffer.java) which is the recommended + (~ 25% faster) reimplementation using direct byte-array and cached field accesses. + * [IoSerialiser](../../microservice/src/main/java/de/gsi/serializer/IoSerialiser.java) which defines the compound wire-format + for more complex objects (e.g. `List`, `Map`, multi-dimensional arrays etc), including field headers, and annotations. + There are three default implementations: + *(N.B. `IoSerialiser` allows further extensions to any other structurally similar protocol. A robust implementation of + [IoSerialiser::checkHeaderInfo()](../../microservice/src/main/java/de/gsi/serializer/IoSerialiser.java#L20) + is critical in order to distinguish new protocols from existing ones.)* + - [BinarySerialiser](../../microservice/src/main/java/de/gsi/serializer/spi/BinarySerialiser.java) which is the primary + binary-based transport protocol used by this library, + - [CmwLightSerialiser](../../microservice/src/main/java/de/gsi/serializer/spi/CmwLightSerialiser.java) which is the backward + compatible re-implementation of an existing proprietary protocol internally used in our facility, and + - [JsonSerialiser](../../microservice/src/main/java/de/gsi/serializer/spi/JsonSerialiser.java) which implements the + [JSON](https://www.json.org/) protocol commonly used in RESTful HTTP-based services. + * [IoClassSerialiser](../../microservice/src/main/java/de/gsi/serializer/IoClassSerialiser.java) which deals with the automatic + mapping and (de-)serialisation between the class field structure and specific wire-format. This class defines default strategies + for generic and nested classes and can be further extended by custom serialiser prototypes for more complex classes, + other custom nested protocols or interfaces using the + [FieldSerialiser](../../microservice/src/main/java/de/gsi/serializer/FieldSerialiser.java) interface. + +A short working example of how these can be used is shown in [IoClassSerialiserSimpleTest](../../microservice/src/test/java/de/gsi/serializer/IoClassSerialiserSimpleTest.java): +```Java +@Test +void simpleTest() { + final IoBuffer byteBuffer = new FastByteBuffer(10_000); // alt: new ByteBuffer(10_000); + final IoClassSerialiser ioClassSerialiser = new IoClassSerialiser(byteBuffer, BinarySerialiser.class); + TestDataClass data = new TestDataClass(); // object to be serialised + + byteBuffer.reset(); + ioClassSerialiser.serialiseObject(data); // pojo -> serialised data + // [..] stream/write serialised byteBuffer content [..] + + // [..] stream/read serialised byteBuffer content + byteBuffer.flip(); // mark byte-buffer for reading + TestDataClass received = ioClassSerialiser.deserialiseObject(TestDataClass.class); + + // check data equality, etc... + assertEquals(data, received); +} +``` +The specific wire-format that the [IoClassSerialiser](../../microservice/src/main/java/de/gsi/serializer/IoClassSerialiser.java) uses can be set either programmatically or dynamically (auto-detection based on serialised data content header) via: +```Java + ioClassSerialiser.setMatchedIoSerialiser(BinarySerialiser.class); + ioClassSerialiser.setMatchedIoSerialiser(CmwLightSerialiser.class); + ioClassSerialiser.setMatchedIoSerialiser(JsonSerialiser.class); + // to auto-detect the suitable serialiser based on serialised data header: + ioClassSerialiser.setAutoMatchSerialiser(true); +``` +The extension for arbitrary custom classes or interfaces can be achieved through (here for the `DoubleArrayList` class) via: +```Java + serialiser.addClassDefinition(new FieldSerialiser<>( + (io, obj, field) -> field.getField().set(obj, DoubleArrayList.wrap(io.getDoubleArray())), // IoBuffer → class field reader function + (io, obj, field) -> DoubleArrayList.wrap(io.getDoubleArray()), // return function - generates new object based on IoBuffer content + (io, obj, field) -> { // class field → IoBuffer writer function + final DoubleArrayList retVal = (DoubleArrayList) field.getField().get(obj); + io.put(field, retVal.elements(), retVal.size()); + }, + DoubleArrayList.class)); +``` +The [DataSetSerialiser](../../microservice/src/main/java/de/gsi/serializer/spi/iobuffer/DataSetSerialiser.java) serialiser +implementation is a representative example and serialises the [DataSet](../../chartfx-dataset/src/main/java/de/gsi/dataset/DataSet.java) +interface into an abstract implementation-independet wire-format using the [FieldDataSetHelper](../../microservice/src/main/java/de/gsi/serializer/spi/iobuffer/FieldDataSetHelper.java) +function. This is also the most prominent common domain object definition that is used within our MVC-pattern driven microservice-, +data-processing-, and UI-applications and one of the original primary motivations why we designed and built the `IoClassSerialiser` implementation. + +### Primary Serialiser Functionality Goals and Constraints +Some of the aspects that were incorporated into the design, loosely ordered according to their importance: + 1. performance: providing an optimised en-/decoding that minimises the effective total latency between the data object + content being ready to be serialised and sent by the server until the object is received, fully de-serialised and ready + for further processing on the client-side. + *N.B. some serialisers trade-off size for en-/decoding speed, which may be suitable for primarily network-io limited + systems. Since io-bandwidth is not a primary concern for our local network, we chose a rather simple encoder with no + explicit compression stage to save CPU clock cycles.* + 2. facilitate multi-protocol implementations, protocol evolution and loose coupling between data object definitions on + the server- and corresponding client-side, ie. services and clients may communicate with different protocol versions + and need to agree only on a mandatory small sub-set of information they both need to share. + *N.B. most micro-services develop naturally and grow their functionality with time. This decoupling is necessary to + provide a smooth and soft upgrade path for early adopters that require these new functionalities (ie. thus also being updated + during regular facility operation), and clients that may require a controlled maintenance period, e.g. safety related systems, + that need a formal qualification process prior to being deployed into regular operation with a modified data-model.* + 3. same client- and server-side API, decoupling the serialisers' wire-formats (ie. different binary formats, JSON, XML, YML, ...) + from the specific microservice APIs and low-level transport protocols that transmit the serialised data + *N.B. encapsulates domain-specific control as well as the generic microservice logic into reusable code blocks that + can be re-implemented if deemed necessary, and that are decoupled from the specific required io-formats, which are usually + either driven by technical necessity (e.g. device supporting only one data wire-format) and/or client-side preferences + (e.g. web-based clients typically favouring RESTful JSON-based protocols while high-throughput clients with real-time requirements + often favour more optimised binary data protocols over TCP/UDP-based sockets).* + 4. derive schemas for generic data directly from C++ or Java class structures and basic types rather than a 3rd-party IDL + definition (ie. using [Pocos](https://en.wikipedia.org/wiki/Plain_Old_C%2B%2B_Object) & [Pojos](https://en.wikipedia.org/wiki/Plain_old_Java_object) as IDL) + - aims at a high compatibility between C++, Java and other languages derived thereof and leverages existing experience + of developers with those languages + *N.B. this improves the productivity of new/occasional/less-experienced users who need to be ony vaguely familiar + with C++/Java and do not need to learn yet another new dedicated DSL. This also inverts the problem: rather than + 'here are the data structures you allowed to use to be serialised' to 'what can be done to serialise the structures + one already is using'.* + - enforces stronger type-safety + *N.B. some other serialisers encode only sub-sets of the possible data structures, and or reduce the specific type + to encompassing super types. For example, integer-types such as `byte`, `short`, `int` all being mapped to `long`, + or all floating-point-type numbers to `double` which due to the ambiguity causes unnecessary numerical decoding errors + on the deserialisation side.* + - support for simple data primitives, nested class objects or common data container, such as `Collection`, `List`, + `Set`, ..., `Map`, etc. + *N.B. We found, that due to the evolution of our microservices and data protocol definitions, we frequently had to + remap and rewrite adapters between our internal map-based data-formats and class objects which proved to be a frequent + and unnecessary source of coding errors.* + - efficient (first-class) support of large collections of numeric (floating-point) data + *N.B. many of the serialiser supporting binary wire-format seem to be optimised for simple data structure that are typically + much smaller than 1k Bytes rather than large numeric arrays that were eiter slow to encode and/or required custom serialiser + extensions.* + - usage of compile-time reflection ie. offline/one-time optimisation prior to running → run deterministic/optimally while + online w/o relying on dynamic parsing optimisations + *N.B. this particularly simplifies the evolution, modification of data structures, and removes one of the common source + of coding errors, since the synchronisation between class-structure, serialised-data-structure and formal-IDL-structure + is omitted.* + - optional: support run-time reflection as a fall-back solution for new data/users that haven't used the compile-time reflection + - optional support of UTF-8-based and fall-back to ISO8859-1-based String encoding if a faster or more efficient en-/decoding is needed. + 5. allow schema extensions through optional custom (de-)serialiser routines for known classes or interface that are either more optimised, + or that implement a specific data-exchange format for a given generic class interface. + 6. self-documented data-structures with optional data field annotations to document and communicate the data-exchange-API-intend to the client + - some examples: 'unit' and 'description' of specific data fields, read/write field access definitions, definition of field sub-sets, etc. + - allows on-the-fly data structure (full schema) documentation for users based on the transmitted wire-format structure + w/o the explicite need to have access to the exact service class domain object definition + *N.B. we keep the code public, this also facilitate automatic documentation updates whenever the code is being modified + and opens the possibility of [OpenAPI specification](https://swagger.io/specification/) -style extensions common for RESTful service.* + - optional: full schema information is transmitted only for the first and (optionally) suppressed in subsequent transmissions for improved performance. + *N.B. trade-off between optimise latency/throughput in high-volume paths vs. feature-rich/documented data storage protocol for + less critical low-volume 'get/set' operations.* + 7. minimise code-base and code-bloat -- for two reasons: + - smaller code usually leads to smaller compiled binary sizes that are more likely to fit into CPU cache, thus are less + likely to be evicted on context changes, and result into overall faster code. + *N.B. while readability is an important issue, we found that certain needless use of 'interface + impl pattern' + (ie. only one implementation for given per interface) are harder to read and harder to optimise for the (JIT) compiler too. + As an example, in-lining and keeping the code in one (albeit larger) source file proved to yield much faster results + for the `CmwLightSerialiser` reimplementation of an existing internally used wire-format.* + - maintenance: code should be able to be re-engineered or optimised within typically 2 weeks by one skilled developer. + *N.B. more code requires more time to read and to understand. While there are many skilled developer, having a simple + code base also implies that the code can be more easily be modified, tested, fixed or maintained by any internally + available developer. Also, be believe that this makes it possibly more likely to be adopted by external users that + want to understand, upgrade, or bug-fix of 'what is under the hood' and is of specific interest to them. Having too + many paradigms, patterns or library dependencies -- even with modern IDEs -- makes it unnecessarily hard for new or + occasional users for getting started.* + 8. unit-test driven development + *N.B. this to minimise errors, loop-holes, and to detect potential regression early-on as part of a general CI/CD strategy, + but also to continuously re-evaluate design choices and quantitative evolution of the performance (for both: potential + regressions and/or improvements, if possible).* + 9. free- and open-source code basis w/o strings-attached: + - it is important to us that this code can be re-used, built- and improved-upon by anybody and not limited by + unnecessary hurdles to due proprietary or IP-protected interfaces or licenses. + *N.B. we chose the [LGPLv3](https://www.gnu.org/licenses/lgpl-3.0.txt) license in order that this remains free for future use, + and to foster evolution of ideas and further developments that build upon this. See also [this](https://github.com/GSI-CS-CO/chart-fx/issues/221).* + +### Some Serialiser Performance Comparison Results +The following examples are qualitative and primarily used to verify that our implementation is not significantly slower than +another reference implementation and to document possible performance regression when refactoring the code base. +Example output of [SerialiserQuickBenchmark.java](../src/test/java/de/gsi/serializer/benchmark/SerialiserQuickBenchmark.java) which compares the +map-only, custom and full-pojo-to-pojo (de-)serialisation performance for the given low-level wire-format: + +```text +Example output - numbers should be compared relatively (nIterations = 100000): +(openjdk 11.0.7 2020-04-14, ASCII-only, nSizePrimitiveArrays = 10, nSizeString = 100, nestedClassRecursion = 1) +[..] more string-heavy TestDataClass +- run 1 +- JSON Serializer (Map only) throughput = 371.4 MB/s for 5.2 kB per test run (took 1413.0 ms) +- CMW Serializer (Map only) throughput = 220.2 MB/s for 6.3 kB per test run (took 2871.0 ms) +- CmwLight Serializer (Map only) throughput = 683.1 MB/s for 6.4 kB per test run (took 935.0 ms) +- IO Serializer (Map only) throughput = 810.0 MB/s for 7.4 kB per test run (took 908.0 ms) + +- FlatBuffers (custom FlexBuffers) throughput = 173.7 MB/s for 6.1 kB per test run (took 3536.0 ms) +- CmwLight Serializer (custom) throughput = 460.5 MB/s for 6.4 kB per test run (took 1387.0 ms) +- IO Serializer (custom) throughput = 545.0 MB/s for 7.3 kB per test run (took 1344.0 ms) + +- JSON Serializer (POJO) throughput = 53.8 MB/s for 5.2 kB per test run (took 9747.0 ms) +- CMW Serializer (POJO) throughput = 182.8 MB/s for 6.3 kB per test run (took 3458.0 ms) +- CmwLight Serializer (POJO) throughput = 329.2 MB/s for 6.3 kB per test run (took 1906.0 ms) +- IO Serializer (POJO) throughput = 374.9 MB/s for 7.2 kB per test run (took 1925.0 ms) + +[..] more primitive-array-heavy TestDataClass +(openjdk 11.0.7 2020-04-14, UTF8, nSizePrimitiveArrays = 1000, nSizeString = 0, nestedClassRecursion = 0) +- run 1 +- JSON Serializer (Map only) throughput = 350.7 MB/s for 34.3 kB per test run (took 9793.0 ms) +- CMW Serializer (Map only) throughput = 1.7 GB/s for 29.2 kB per test run (took 1755.0 ms) +- CmwLight Serializer (Map only) throughput = 6.7 GB/s for 29.2 kB per test run (took 437.0 ms) +- IO Serializer (Map only) throughput = 6.1 GB/s for 29.7 kB per test run (took 485.0 ms) + +- FlatBuffers (custom FlexBuffers) throughput = 123.1 MB/s for 30.1 kB per test run (took 24467.0 ms) +- CmwLight Serializer (custom) throughput = 3.9 GB/s for 29.2 kB per test run (took 751.0 ms) +- IO Serializer (custom) throughput = 3.8 GB/s for 29.7 kB per test run (took 782.0 ms) + +- JSON Serializer (POJO) throughput = 31.7 MB/s for 34.3 kB per test run (took 108415.0 ms) +- CMW Serializer (POJO) throughput = 1.5 GB/s for 29.2 kB per test run (took 1924.0 ms) +- CmwLight Serializer (POJO) throughput = 3.5 GB/s for 29.1 kB per test run (took 824.0 ms) +- IO Serializer (POJO) throughput = 3.4 GB/s for 29.7 kB per test run (took 870.0 ms) +``` + +A more thorough test using the Java micro-benchmark framework [JMH](https://openjdk.java.net/projects/code-tools/jmh/) output +of [SerialiserBenchmark.java](../src/test/java/de/gsi/serializer/benchmark/SerialiserBenchmark.java) for a string-heavy and +for a numeric-data-heavy test data class: + +```text +Benchmark (testClassId) Mode Cnt Score Error Units +SerialiserBenchmark.customCmwLight string-heavy thrpt 10 49954.479 ± 560.726 ops/s +SerialiserBenchmark.customCmwLight numeric-heavy thrpt 10 22433.828 ± 195.939 ops/s +SerialiserBenchmark.customFlatBuffer string-heavy thrpt 10 18446.085 ± 71.311 ops/s +SerialiserBenchmark.customFlatBuffer numeric-heavy thrpt 10 233.869 ± 7.314 ops/s +SerialiserBenchmark.customIoSerialiser string-heavy thrpt 10 53638.035 ± 367.122 ops/s +SerialiserBenchmark.customIoSerialiser numeric-heavy thrpt 10 24277.732 ± 200.380 ops/s +SerialiserBenchmark.customIoSerialiserOptim string-heavy thrpt 10 79759.984 ± 799.944 ops/s +SerialiserBenchmark.customIoSerialiserOptim numeric-heavy thrpt 10 24192.169 ± 419.019 ops/s +SerialiserBenchmark.customJson string-heavy thrpt 10 17619.026 ± 250.917 ops/s +SerialiserBenchmark.customJson numeric-heavy thrpt 10 138.461 ± 2.972 ops/s +SerialiserBenchmark.mapCmwLight string-heavy thrpt 10 79273.547 ± 2487.931 ops/s +SerialiserBenchmark.mapCmwLight numeric-heavy thrpt 10 67374.131 ± 954.149 ops/s +SerialiserBenchmark.mapIoSerialiser string-heavy thrpt 10 81295.197 ± 2391.616 ops/s +SerialiserBenchmark.mapIoSerialiser numeric-heavy thrpt 10 67701.564 ± 1062.641 ops/s +SerialiserBenchmark.mapIoSerialiserOptimized string-heavy thrpt 10 115008.285 ± 2390.426 ops/s +SerialiserBenchmark.mapIoSerialiserOptimized numeric-heavy thrpt 10 68879.735 ± 1403.197 ops/s +SerialiserBenchmark.mapJson string-heavy thrpt 10 14474.142 ± 1227.165 ops/s +SerialiserBenchmark.mapJson numeric-heavy thrpt 10 163.928 ± 0.968 ops/s +SerialiserBenchmark.pojoCmwLight string-heavy thrpt 10 41821.232 ± 217.594 ops/s +SerialiserBenchmark.pojoCmwLight numeric-heavy thrpt 10 33820.451 ± 568.264 ops/s +SerialiserBenchmark.pojoIoSerialiser string-heavy thrpt 10 41899.128 ± 940.030 ops/s +SerialiserBenchmark.pojoIoSerialiser numeric-heavy thrpt 10 33918.815 ± 376.551 ops/s +SerialiserBenchmark.pojoIoSerialiserOptim string-heavy thrpt 10 53811.486 ± 920.474 ops/s +SerialiserBenchmark.pojoIoSerialiserOptim numeric-heavy thrpt 10 32463.267 ± 635.326 ops/s +SerialiserBenchmark.pojoJson string-heavy thrpt 10 23327.701 ± 288.871 ops/s +SerialiserBenchmark.pojoJson numeric-heavy thrpt 10 161.396 ± 3.040 ops/s +SerialiserBenchmark.pojoJsonCodeGen string-heavy thrpt 10 23586.818 ± 470.233 ops/s +SerialiserBenchmark.pojoJsonCodeGen numeric-heavy thrpt 10 163.250 ± 1.254 ops/s +``` +*N.B. The 'FlatBuffer' implementation is bit of an outlier and uses internally FlatBuffer's `FlexBuffer` builder which does not +support or is optimised for large primitive arrays. `FlexBuffer` was chosen primarily for comparison since it supported flexible +compile/run-time map-type structures similar to the other implementations, whereas the faster Protobuf and Flatbuffer builder +require IDL-based desciptions that are used during compile-time to generate the necessary data-serialiser stubs.* + +JSON-compatible strings are easy to construct and write. Nevertheless, we chose the [Json-Itererator](https://github.com/json-iterator/java) +library as backend for implementing the [JsonSerialiser](../../microservice/src/main/java/de/gsi/serializer/spi/JsonSerialiser.java) +for purely pragmatic reasons and to initially avoid common pitfalls in implementing a robust JSON deserialiser. +The [JsonSelectionBenchmark.java](../src/test/java/de/gsi/serializer/benchmark/JsonSelectionBenchmark.java) compares the +choice with several other commonly used JSON serialisation libraries for a string-heavy and a numeric-data-heavy test data class: +```text +Benchmark (testClassId) Mode Cnt Score Error Units + JsonSelectionBenchmark.pojoFastJson string-heavy thrpt 10 12857.850 ± 109.050 ops/s + JsonSelectionBenchmark.pojoFastJson numeric-heavy thrpt 10 91.458 ± 0.437 ops/s + JsonSelectionBenchmark.pojoGson string-heavy thrpt 10 6253.698 ± 50.267 ops/s + JsonSelectionBenchmark.pojoGson numeric-heavy thrpt 10 48.215 ± 0.265 ops/s + JsonSelectionBenchmark.pojoJackson string-heavy thrpt 10 16563.604 ± 244.329 ops/s + JsonSelectionBenchmark.pojoJackson numeric-heavy thrpt 10 135.780 ± 1.074 ops/s + JsonSelectionBenchmark.pojoJsonIter string-heavy thrpt 10 10733.539 ± 35.605 ops/s + JsonSelectionBenchmark.pojoJsonIter numeric-heavy thrpt 10 86.629 ± 1.122 ops/s + JsonSelectionBenchmark.pojoJsonIterCodeGen string-heavy thrpt 10 41048.034 ± 396.628 ops/s + JsonSelectionBenchmark.pojoJsonIterCodeGen numeric-heavy thrpt 10 377.412 ± 9.755 ops/s +``` +Performance was not of primary concern for us, since JSON-based protocols are anyway slow. The heavy penalty for +numeric-heavy data is largely related to the inefficient string representation of double values. + +### References + + * Brian Goetz, "[Towards Better Serialization](https://cr.openjdk.java.net/~briangoetz/amber/serialization.html)", 2019 + * Pieter Hintjens et al., "[ZeroMQ's ZGuide on Serialisation](http://zguide.zeromq.org/page:chapter7#Serializing-Your-Data)" in 'ØMQ - The Guide', 2020 + * N. Trofimov et al., "[Remote Device Access in the new CERN accelerator controls middleware](https://accelconf.web.cern.ch/ica01/papers/THAP003.pdf)", ICALEPCS 2001, San Jose, USA, 2001, (proprietary, closed-source) + * J. Lauener, W. Sliwinski, "[How to Design & Implement a Modern Communication Middleware based on ZeroMQ](https://cds.cern.ch/record/2305650/files/mobpl05.pdf)", ICALEPS2017, Barcelona, Spain, 2017, (proprietary, closed-source) + * [Google's own Protobuf Serialiser](https://github.com/protocolbuffers/protobuf) + * [Google's own FlatBuffer Serialiser](https://github.com/google/flatbuffers) + * [Implementing High Performance Parsers in Java](https://www.infoq.com/articles/HIgh-Performance-Parsers-in-Java-V2/) + * [Is Protobuf 5x Faster Than JSON?](https://dzone.com/articles/is-protobuf-5x-faster-than-json-part-ii) ([Part 1](https://dzone.com/articles/is-protobuf-5x-faster-than-json), [Part 2](https://dzone.com/articles/is-protobuf-5x-faster-than-json-part-ii)) and reference therein + + diff --git a/docs/MajordomoProtocol.md b/docs/MajordomoProtocol.md new file mode 100644 index 00000000..b6e3f402 --- /dev/null +++ b/docs/MajordomoProtocol.md @@ -0,0 +1,11 @@ +# Majordomo Protocol (MDP) TL;DR; Comparison + +Brief summary of definitions in: + * MDP v 0.1: https://rfc.zeromq.org/spec/7/ + * MDP v 0.2: https://rfc.zeromq.org/spec/18/ + * Majordomo Management Interface (MMI): https://rfc.zeromq.org/spec/8/ + +and OpenCMW protocol extension proposal ([pdf](Majordomo_protocol_comparison.pdf), [spreadsheet source](Majordomo_protocol_comparison.ods)): +![Majordomo Comparison](Majordomo_protocol_comparison.png) + + \ No newline at end of file diff --git a/docs/Majordomo_protocol_comparison.ods b/docs/Majordomo_protocol_comparison.ods new file mode 100644 index 00000000..88b4d143 Binary files /dev/null and b/docs/Majordomo_protocol_comparison.ods differ diff --git a/docs/Majordomo_protocol_comparison.pdf b/docs/Majordomo_protocol_comparison.pdf new file mode 100644 index 00000000..63708722 Binary files /dev/null and b/docs/Majordomo_protocol_comparison.pdf differ diff --git a/docs/Majordomo_protocol_comparison.png b/docs/Majordomo_protocol_comparison.png new file mode 100644 index 00000000..87396f4e Binary files /dev/null and b/docs/Majordomo_protocol_comparison.png differ diff --git a/formatFiles.sh b/formatFiles.sh new file mode 100755 index 00000000..d042f7cd --- /dev/null +++ b/formatFiles.sh @@ -0,0 +1,58 @@ +#/bin/bash + +#enforces .clang-format style guide prior to committing to the git repository + +CLANG_MIN_VERSION="9.0.0" + +set -e + +CLANG_FORMAT="$(command -v clang-format)" +CLANG_VERSION="$(${CLANG_FORMAT} --version | sed '/^clang-format version /!d;s///;s/-.*//;s///g')" + +compare_version () { + echo " " + if [[ $1 == $2 ]] + then + CLANG_MIN_VERSION_MATCH="=" + return + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + CLANG_MIN_VERSION_MATCH="<" + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + CLANG_MIN_VERSION_MATCH=">" + return + fi + done + CLANG_MIN_VERSION_MATCH="=" + return +} + +compare_version ${CLANG_MIN_VERSION} ${CLANG_VERSION} + +files=$((git status -uall --porcelain | awk 'match($1, "M??"){print $2}' | grep -Ei "\.(c|cc|cpp|cxx|c\+\+|h|hh|hpp|hxx|h\+\+|java)$") || true) +if [ -n "${files}" ]; then + + if [ -n "${CLANG_FORMAT}" ] && [ "$CLANG_MIN_VERSION_MATCH" != "<" ]; then + spaced_files=$(echo "$files" | paste -s -d " " -) + # echo "reformatting ${spaced_files}" + "${CLANG_FORMAT}" -style=file -i $spaced_files >/dev/null + fi +fi diff --git a/formatLastCommit.sh b/formatLastCommit.sh new file mode 100755 index 00000000..2d9948db --- /dev/null +++ b/formatLastCommit.sh @@ -0,0 +1,62 @@ +#/bin/bash + +#enforces .clang-format style guide prior to committing to the git repository + +CLANG_MIN_VERSION="9.0.0" + +set -e + +CLANG_FORMAT="$(command -v clang-format)" +CLANG_VERSION="$(${CLANG_FORMAT} --version | sed '/^clang-format version /!d;s///;s/-.*//;s///g')" + +compare_version () { + echo " " + if [[ $1 == $2 ]] + then + CLANG_MIN_VERSION_MATCH="=" + return + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + CLANG_MIN_VERSION_MATCH="<" + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + CLANG_MIN_VERSION_MATCH=">" + return + fi + done + CLANG_MIN_VERSION_MATCH="=" + return +} + +compare_version ${CLANG_MIN_VERSION} ${CLANG_VERSION} +git reset HEAD~1 --soft + +files=$((git diff --name-only --cached | grep -Ei "\.(c|cc|cpp|cxx|c\+\+|h|hh|hpp|hxx|h\+\+|java)$") || true) +if [ -n "${files}" ]; then + + if [ -n "${CLANG_FORMAT}" ] && [ "$CLANG_MIN_VERSION_MATCH" != "<" ]; then + spaced_files=$(echo "$files" | paste -s -d " " -) + echo "reformatting ${spaced_files}" + "${CLANG_FORMAT}" -style=file -i $spaced_files >/dev/null + git --no-pager diff + git add ${spaced_files} + fi +fi +git commit -C ORIG_HEAD diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..cbd56cda --- /dev/null +++ b/pom.xml @@ -0,0 +1,368 @@ + + 4.0.0 + io.opencmw + opencmw + + ${revision}${sha1}${changelist} + pom + opencmw + + Microservice middleware framework for beam-based feedback systems at the FAIR particle accelerator. + + + + serialiser + core + server + server-rest + client + concepts + + + + 0.0.1 + -SNAPSHOT + + UTF-8 + 11 + 11 + 2.0.0-alpha0 + 20.1.0 + 0.5.2 + 3.4.2 + 3.13.3 + 9.4.35.v20201120 + 0.9.23 + 8.5.2 + 3.27.0-GA + 4.9.1 + 3.11 + 5.7.1 + 4.0.3 + 0.5.0 + 1.27 + 1.6.3 + 2.2 + 11.2.3 + + + + + LGPLv3 + https://www.gnu.org/licenses/lgpl-3.0.html + + + + + GSI Helmholtzzentrum für Schwerionenforschung GmbH + http://www.gsi.de + + + https://github.com/fair-acc/opencmw-java + + + + rstein + Ralph J. Steinhagen + R.Steinhagen@gsi.de + https://fair-wiki.gsi.de/FC2WG + +1 + + owner + architect + developer + + + + akrimm + Alexander Krimm + A.Krimm@gsi.de + +1 + + owner + architect + developer + + + + + + scm:git:https://github.com/fair-acc/opencmw-java + scm:git:git@github.com:fair-acc/opencmw-java + https://github.com/fair-acc/opencmw-java + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + 11 + + + + org.codehaus.mojo + flatten-maven-plugin + 1.2.5 + + true + resolveCiFriendliesOnly + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + + ${argLine} + --add-opens io.opencmw.serialiser/io.opencmw.serialiser=ALL-UNNAMED + --add-opens io.opencmw.serialiser/io.opencmw.serialiser.spi=ALL-UNNAMED + --add-opens io.opencmw.serialiser/io.opencmw.serialiser.spi.helper=ALL-UNNAMED + --add-opens io.opencmw.serialiser/io.opencmw.serialiser.spi.iobuffer=ALL-UNNAMED + --add-opens io.opencmw.serialiser/io.opencmw.serialiser.annotations=ALL-UNNAMED + --add-opens io.opencmw/io.opencmw=ALL-UNNAMED + --add-opens io.opencmw/io.opencmw.filter=ALL-UNNAMED + --add-opens io.opencmw/io.opencmw.utils=ALL-UNNAMED + --add-opens io.opencmw.client/io.opencmw.client=ALL-UNNAMED + --add-opens io.opencmw.client/io.opencmw.client.rest=ALL-UNNAMED + --add-opens io.opencmw.client/io.opencmw.client.cmwlight=ALL-UNNAMED + --add-opens io.opencmw.concepts/io.opencmw.concepts.cmwlight=ALL-UNNAMED + --add-opens io.opencmw.concepts/io.opencmw.concepts.aggregate=ALL-UNNAMED + -Duser.language=en -Duser.country=US + -Xms256m -Xmx2048m -XX:G1HeapRegionSize=32m + -Djava.awt.headless=true -Dtestfx.robot=glass -Dtestfx.headless=true -Dprism.order=sw + + 1 + 2 + random + + + + org.jacoco + jacoco-maven-plugin + 0.8.6 + + + + prepare-agent + + + + report + test + + report + + + + + + + + + + org.slf4j + slf4j-api + ${version.slf4j} + + + org.slf4j + slf4j-simple + ${version.slf4j} + test + + + org.jetbrains + annotations + ${version.jetbrains.annotations} + compile + + + org.junit.jupiter + junit-jupiter-api + ${version.jupiter} + test + + + org.junit.jupiter + junit-jupiter-engine + ${version.jupiter} + test + + + org.junit.jupiter + junit-jupiter-params + ${version.jupiter} + test + + + org.awaitility + awaitility + ${version.awaitility} + test + + + de.sandec + JMemoryBuddy + ${version.JMemoryBuddy} + test + + + + org.openjdk.jmh + jmh-core + ${version.jmh} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${version.jmh} + test + + + + + + releaseGithub + + + release + github + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + --pinentry-mode + loopback + + + + + + + + github + GSI Github repository + https://maven.pkg.github.com/fair-acc/opencmw-java + + + + + releaseOSSRH + + + release + ossrh + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + --pinentry-mode + loopback + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + true + + ossrh + https://oss.sonatype.org/ + true + + + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + + + diff --git a/ruleset.xml b/ruleset.xml new file mode 100644 index 00000000..47febc43 --- /dev/null +++ b/ruleset.xml @@ -0,0 +1,103 @@ + + + + PMD rules for CSCOAP at GSI Java Applications + + + + + + .*\/generated-sources\/.* + + .*\/src\/test\/.* + + .*\/target\/.* + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5 + + diff --git a/serialiser/pom.xml b/serialiser/pom.xml new file mode 100644 index 00000000..ad5e3b86 --- /dev/null +++ b/serialiser/pom.xml @@ -0,0 +1,71 @@ + + + 4.0.0 + + + io.opencmw + opencmw + ${revision}${sha1}${changelist} + ../pom.xml + + + serialiser + + + Efficient reflection based serialisers. + + + + + de.gsi.dataset + chartfx-dataset + ${version.chartfx} + true + + + + it.unimi.dsi + fastutil + ${version.fastutil} + + + com.jsoniter + jsoniter + ${version.jsoniter} + + + org.javassist + javassist + ${version.javassist} + + + + + + com.google.flatbuffers + flatbuffers-java + 1.12.0 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.12.1 + test + + + com.google.code.gson + gson + 2.8.6 + test + + + com.alibaba + fastjson + 1.2.75 + test + + + + diff --git a/serialiser/src/main/java/io/opencmw/serialiser/Cat.java b/serialiser/src/main/java/io/opencmw/serialiser/Cat.java new file mode 100644 index 00000000..a5d272cc --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/Cat.java @@ -0,0 +1,12 @@ +package io.opencmw.serialiser; + +/** + * private type inner categories + * + * @author rstein + */ +enum Cat { + SINGLE_VALUE, + ARRAY, + COMPLEX_OBJECT +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/DataType.java b/serialiser/src/main/java/io/opencmw/serialiser/DataType.java new file mode 100644 index 00000000..cac68f8d --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/DataType.java @@ -0,0 +1,218 @@ +package io.opencmw.serialiser; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +import io.opencmw.serialiser.spi.BinarySerialiser; + +import de.gsi.dataset.spi.utils.MultiArrayBoolean; +import de.gsi.dataset.spi.utils.MultiArrayByte; +import de.gsi.dataset.spi.utils.MultiArrayChar; +import de.gsi.dataset.spi.utils.MultiArrayDouble; +import de.gsi.dataset.spi.utils.MultiArrayFloat; +import de.gsi.dataset.spi.utils.MultiArrayInt; +import de.gsi.dataset.spi.utils.MultiArrayLong; +import de.gsi.dataset.spi.utils.MultiArrayObject; +import de.gsi.dataset.spi.utils.MultiArrayShort; + +/** + * Enum definition for data primitives in the context of serialisation and includes definitions for: + *
    + *
  • primitives (byte, short, ..., float, double, and String), and + *
  • arrays thereof (ie. byte[], short[], ..., float[], double[], and String[]), as well as + *
  • complex objects implementing Collections (ie. Set, List, Queues), Enums or Maps. + *
+ * Any other complex data objects can be stored/extended using the {@link DataType#OTHER OTHER} sub-type. + * + *

+ * N.B. Multi-dimensional arrays are handled through one-dimensional striding arrays with the additional + * infos on number of dimensions and size for each individual dimension. + * + * @author rstein + * @see BinarySerialiser + * @see striding arrays + */ +public enum DataType { + // @formatter:off + // clang-format off + // start marker + START_MARKER(0, "start_marker", "", 0, Cat.SINGLE_VALUE), + // primitive types + BOOL(1, "bool", "boolean", 1, Cat.SINGLE_VALUE, boolean.class, Boolean.class), + BYTE(2, "byte", "byte", 1, Cat.SINGLE_VALUE, byte.class, Byte.class), + SHORT(3, "short", "short", 2, Cat.SINGLE_VALUE, short.class, Short.class), + INT(4, "int", "int", 4, Cat.SINGLE_VALUE, int.class, Integer.class), + LONG(5, "long", "long", 8, Cat.SINGLE_VALUE, long.class, Long.class), + FLOAT(6, "float", "float", 4, Cat.SINGLE_VALUE, float.class, Float.class), + DOUBLE(7, "double", "double", 8, Cat.SINGLE_VALUE, double.class, Double.class), + CHAR(8, "char", "char", 2, Cat.SINGLE_VALUE, char.class, Character.class), + STRING(9, "string", "java.lang.String", 1, Cat.ARRAY, String.class, String.class), + + // array of primitive types + BOOL_ARRAY(101, "bool_array", "[Z", 1, Cat.ARRAY, boolean[].class, Boolean[].class, MultiArrayBoolean.class), + BYTE_ARRAY(102, "byte_array", "[B", 1, Cat.ARRAY, byte[].class, Byte[].class, MultiArrayByte.class), + SHORT_ARRAY(103, "short_array", "[S", 2, Cat.ARRAY, short[].class, Short[].class, MultiArrayShort.class), + INT_ARRAY(104, "int_array", "[I", 4, Cat.ARRAY, int[].class, Integer[].class, MultiArrayInt.class), + LONG_ARRAY(105, "long_array", "[J", 8, Cat.ARRAY, long[].class, Long[].class, MultiArrayLong.class), + FLOAT_ARRAY(106, "float_array", "[F", 4, Cat.ARRAY, float[].class, Float[].class, MultiArrayFloat.class), + DOUBLE_ARRAY(107, "double_array", "[D", 8, Cat.ARRAY, double[].class, Double[].class, MultiArrayDouble.class), + CHAR_ARRAY(108, "char_array", "[C", 2, Cat.ARRAY, char[].class, Character[].class, MultiArrayChar.class), + STRING_ARRAY(109, "string_array", "[java.lang.String", 1, Cat.ARRAY, String[].class, String[].class, MultiArrayObject.class), + + // complex objects + ENUM(201, "enum", "java.lang.Enum", 4, Cat.ARRAY, Enum.class), + LIST(202, "list", "", 1, Cat.ARRAY, List.class), + MAP(203, "map", "", 1, Cat.ARRAY, Map.class), + QUEUE(204, "queue", "", 1, Cat.ARRAY, Queue.class), + SET(205, "set", "", 1, Cat.ARRAY, Set.class), + COLLECTION(200, "collection", "", 1, Cat.ARRAY, Collection.class), + + /** default for any other complex or object-type custom data structure, usually followed/refined + * by an additional user-provided custom type ID */ + OTHER(0xFD, "other", "", 1, Cat.COMPLEX_OBJECT, Object.class), + // end marker + END_MARKER(0xFE, "end_marker", "", 0, Cat.SINGLE_VALUE); + // clang-format on + // @formatter:on + + private final int uniqueID; + private final int primitiveSize; + private final String stringValue; + private final String javaName; + private final List> classTypes; + private final boolean scalar; + private final boolean array; + private final boolean object; + + DataType(final int uniqueID, final String stringValue, final String javaName, final int primitiveSize, + final Cat type, final Class... classType) { + this.uniqueID = uniqueID; + this.stringValue = stringValue; + this.javaName = javaName; + this.primitiveSize = primitiveSize; + classTypes = Arrays.asList(classType); + scalar = type.equals(Cat.SINGLE_VALUE); + array = type.equals(Cat.ARRAY); + object = type.equals(Cat.COMPLEX_OBJECT); + } + + /** + * Returns the uniqueID representation of the data type. + * + * @return the uniqueID representation + */ + public int getID() { + return uniqueID; + } + + /** + * Returns the string representation of the data type. + * + * @return the string representation + */ + public String getAsString() { + return stringValue; + } + + /** + * Returns the corresponding java class type matching the given data type + * + * @return the matching java class type + */ + public List> getClassTypes() { + return classTypes; + } + + /** + * Returns the string representation of the java class type. + * + * @return the string representation of the class + */ + public String getJavaName() { + return javaName; + } + + public int getPrimitiveSize() { + return primitiveSize; + } + + public boolean isArray() { + return array; + } + + public boolean isObject() { + return object; + } + + public boolean isScalar() { + return scalar; + } + + /** + * Returns the data type matching the given uniqueID representation, if any. + * + * @param value the value to be searched + * @return the matching data type + */ + public static DataType fromByte(final byte value) { + final int unsignedByte = (value & 0xFF); + for (final DataType type : DataType.values()) { + if (type.uniqueID == unsignedByte) { + return type; + } + } + throw new IllegalArgumentException("byte entry type is not supported: " + value); + } + + /** + * Returns the data type matching the given java class type, if any. + * + * @param classType the value to be searched + * @return the matching data type + */ + public static DataType fromClassType(final Class classType) { + for (final DataType dataType : DataType.values()) { + for (Class type : dataType.getClassTypes()) { + if (type.isAssignableFrom(classType)) { + return dataType; + } + } + } + + throw new IllegalArgumentException("data type not implemented " + classType.getSimpleName()); + } + + /** + * Returns the data type matching the given java string representation, if any. + * + * @param str the string to be searched + * @return the matching data type + */ + public static DataType fromJavaTypeString(final String str) { + for (final DataType type : DataType.values()) { + if (type.stringValue.equals(str)) { + return type; + } + } + throw new IllegalArgumentException("java string entry type is not supported: '" + str + "'"); + } + + /** + * Returns the data type matching the given string representation, if any. + * + * @param str the string to be searched + * @return the matching data type + */ + public static DataType fromString(final String str) { + for (final DataType type : DataType.values()) { + if (type.stringValue.equals(str)) { + return type; + } + } + throw new IllegalArgumentException("string entry type is not supported: '" + str + "'"); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/FieldDescription.java b/serialiser/src/main/java/io/opencmw/serialiser/FieldDescription.java new file mode 100644 index 00000000..07a3985f --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/FieldDescription.java @@ -0,0 +1,94 @@ +package io.opencmw.serialiser; + +import java.lang.reflect.Type; +import java.util.List; + +import io.opencmw.serialiser.utils.ClassUtils; + +public interface FieldDescription { + boolean isAnnotationPresent(); + + FieldDescription findChildField(String fieldName); + + FieldDescription findChildField(final int fieldNameHashCode, final String fieldName); + + List getChildren(); + + /** + * @return the data size in bytes stored after the field header + */ + int getDataSize(); + + /** + * @return the offset in bytes from the field start position until the first data object can be read. + * (N.B. equals to 'getFieldstart() + getDataOffset()', the data ends at 'getDataStartOffset() + getDataSize()' + */ + int getDataStartOffset(); + + /** + * @return the buffer byte position from where the first data object can be read + */ + int getDataStartPosition(); + + /** + * @return the stored data type, see {@link DataType} for details + */ + DataType getDataType(); + + /** + * @return optional meta data tag describing the purpose of this data field (N.B. can be empty String) + */ + String getFieldDescription(); + + /** + * Return optional meta data tag describing the 'direction' of this data field. + * The information encodes the source servicedevelopers intend to the receiving user whether the field can be, for example, + * modified (get/set), set-only, or read-only, or attach any other similar information. Encoding/interpretation is + * left ad-lib to the source service developer. + * + * @return optional meta data (N.B. can be empty String). + */ + String getFieldDirection(); + + /** + * @return optional meta data describing the group/set this data field belongs to (N.B. empty String corresponds to 'all') + */ + List getFieldGroups(); + + /** + * @return the data field's name + */ + String getFieldName(); + + /** + * @return the data field name's hashcode (N.B. used for faster identification of the field) + */ + int getFieldNameHashCode(); + + /** + * + * @return buffer position in byte where the data field header starts + */ + int getFieldStart(); + + /** + * @return optional meta data tag describing the field's SI unit or similar (N.B. can be empty String) + */ + String getFieldUnit(); + + /** + * @return for a hierarchical/nested data structure refers to the parent this field belongs to (N.B. can be null if there isn't a parent, e.g. for a root element) + */ + FieldDescription getParent(); + + Type getType(); + + /** + * Prints the class field structure to the logging output for diagnostics purposes starting from this element as a root. + + * N.B. regarding formatting/parsing + * The indentation depth is controlled via {@link ClassUtils#setIndentationNumberOfSpace}. + * The max recursion depth during the class structure parsing is controlled via {@link ClassUtils#setMaxRecursionDepth}. + */ + void printFieldStructure(); +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/FieldSerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/FieldSerialiser.java new file mode 100644 index 00000000..9d40654b --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/FieldSerialiser.java @@ -0,0 +1,172 @@ +package io.opencmw.serialiser; + +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.spi.ClassFieldDescription; + +/** + * default field serialiser implementation. The user needs to provide the reader and writer consumer lambdas to connect + * to the given serialiser back-end implementation. + * @param function return type + * @author rstein + */ +public class FieldSerialiser { + private static final Logger LOGGER = LoggerFactory.getLogger(FieldSerialiser.class); + private final Class classPrototype; + private final List classGenericArguments; + private final String name; + private final String canonicalName; + private final String simpleName; + private final int cachedHashCode; + protected TriConsumer readerFunction; + protected TriConsumer writerFunction; + protected TriFunction returnFunction; + + /** + * + * @param reader consumer executed when reading from the back-end serialiser implementation + * @param returnFunction function that is being executed for returning a new object to the back-end serialiser implementation + * @param writer consumer executed when writing to the back-end serialiser implementation + * @param classPrototype applicable class/interface prototype reference for which the consumers are applicable (e.g. + * example 1: 'List.class' for List<String> or example 2: 'Map.class' for Map<Integer, String>) + * @param classGenericArguments applicable generics definition (e.g. 'String.class' for List<String> or + * 'Integer.class, String.class' resp.) + */ + public FieldSerialiser(final TriConsumer reader, final TriFunction returnFunction, final TriConsumer writer, final Class classPrototype, Class... classGenericArguments) { + if ((reader == null || returnFunction == null || writer == null)) { + LOGGER.atWarn().addArgument(reader).addArgument(writer).log("caution: reader {}, return {} or writer {} is null"); + } + if (classPrototype == null) { + throw new IllegalArgumentException("classPrototype must not be null"); + } + this.readerFunction = reader; + this.returnFunction = returnFunction; + this.writerFunction = writer; + this.classPrototype = classPrototype; + this.classGenericArguments = Arrays.asList(classGenericArguments); + cachedHashCode = IoClassSerialiser.computeHashCode(classPrototype, this.classGenericArguments); + + String genericFieldString = this.classGenericArguments.isEmpty() ? "" : this.classGenericArguments.stream().map(Type::getTypeName).collect(Collectors.joining(", ", "<", ">")); + + canonicalName = classPrototype.getCanonicalName() + genericFieldString; + simpleName = classPrototype.getSimpleName() + IoClassSerialiser.getGenericFieldSimpleTypeString(this.classGenericArguments); + name = "Serialiser for " + canonicalName; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + return this.hashCode() == obj.hashCode(); + } + + /** + * + * @return canonical name of the class/interface description + */ + public String getCanonicalName() { + return canonicalName; + } + + /** + * + * @return class reference + */ + public Class getClassPrototype() { + return classPrototype; + } + + /** + * + * @return class reference to generics arguments + */ + public List getGenericsPrototypes() { + return classGenericArguments; + } + + /** + * + * @return consumer that is being executed for reading from the back-end serialiser implementation + */ + public TriConsumer getReaderFunction() { + return readerFunction; + } + + /** + * + * @return simple name name of the class/interface description + */ + public String getSimpleName() { + return simpleName; + } + + /** + * + * @return consumer that is being executed for writing to the back-end serialiser implementation + */ + public TriConsumer getWriterFunction() { + return writerFunction; + } + + /** + * + * @return function that is being executed for returning a new object to the back-end serialiser implementation + */ + public TriFunction getReturnObjectFunction() { + return returnFunction; + } + + @Override + public int hashCode() { + return cachedHashCode; + } + + @Override + public String toString() { + return name; + } + + /** + * used as lambda expression for user-level code to read/write data into the given serialiser back-end implementation + * + * @author rstein + */ + public interface TriConsumer { + /** + * Performs this operation on the given arguments. + * + * @param ioSerialiser the reference to the calling IoSerialiser + * @param rootObj the specific root object reference the given field is part of + * @param field the description for the given class member, if null then rootObj is written/read directly + */ + void accept(IoSerialiser ioSerialiser, Object rootObj, ClassFieldDescription field); + } + + /** + * used as lambda expression for user-level code to return new object data (read-case) from the given serialiser back-end implementation + * + * @author rstein + * @param generic return type + */ + public interface TriFunction { + /** + * Performs this operation on the given arguments. + * + * @param ioSerialiser the reference to the calling IoSerialiser + * @param rootObj the specific root object reference the given field is part of + * @param field the description for the given class member, if null then rootObj is written/read directly + * @return The value of the field which is either taken from rootObj if present or compatible or newly allocated otherwise + */ + R apply(IoSerialiser ioSerialiser, Object rootObj, ClassFieldDescription field); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/IoBuffer.java b/serialiser/src/main/java/io/opencmw/serialiser/IoBuffer.java new file mode 100644 index 00000000..934c8a80 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/IoBuffer.java @@ -0,0 +1,209 @@ +package io.opencmw.serialiser; + +/** + * Interface definition in line with the jdk Buffer abstract class. This definition is needed to allow for redirect or + * different buffer implementations. + * + * @author rstein + */ +@SuppressWarnings({ "PMD.ExcessivePublicCount", "PMD.TooManyMethods", "PMD.AvoidUsingShortType" }) // NOPMD - these are short-hand convenience methods +public interface IoBuffer extends IoBufferHeader { + /** + * @return underlying raw byte[] array buffer (if available) + */ + byte[] elements(); + + boolean getBoolean(int position); + + boolean getBoolean(); // NOPMD - nomen est omen + + default boolean[] getBooleanArray() { + return getBooleanArray(null, 0); + } + + default boolean[] getBooleanArray(final boolean[] dst) { + return getBooleanArray(dst, dst == null ? -1 : dst.length); + } + + boolean[] getBooleanArray(final boolean[] dst, final int length); + + byte getByte(int position); + + byte getByte(); + + default byte[] getByteArray() { + return getByteArray(null, 0); + } + + default byte[] getByteArray(final byte[] dst) { + return getByteArray(dst, dst == null ? -1 : dst.length); + } + + byte[] getByteArray(final byte[] dst, final int length); + + char getChar(int position); + + char getChar(); + + default char[] getCharArray() { + return getCharArray(null, 0); + } + + default char[] getCharArray(final char[] dst) { + return getCharArray(dst, dst == null ? -1 : dst.length); + } + + char[] getCharArray(final char[] dst, final int length); + + double getDouble(int position); + + double getDouble(); + + default double[] getDoubleArray() { + return getDoubleArray(null, 0); + } + + default double[] getDoubleArray(final double[] dst) { + return getDoubleArray(dst, dst == null ? -1 : dst.length); + } + + double[] getDoubleArray(final double[] dst, final int length); + + float getFloat(int position); + + float getFloat(); + + default float[] getFloatArray() { + return getFloatArray(null, 0); + } + + default float[] getFloatArray(final float[] dst) { + return getFloatArray(dst, dst == null ? -1 : dst.length); + } + + float[] getFloatArray(final float[] dst, final int length); + + int getInt(int position); + + int getInt(); + + default int[] getIntArray() { + return getIntArray(null, 0); + } + + default int[] getIntArray(final int[] dst) { + return getIntArray(dst, dst == null ? -1 : dst.length); + } + + int[] getIntArray(final int[] dst, final int length); + + long getLong(int position); + + long getLong(); + + default long[] getLongArray() { + return getLongArray(null, 0); + } + + default long[] getLongArray(final long[] dst) { + return getLongArray(dst, dst == null ? -1 : dst.length); + } + + long[] getLongArray(final long[] dst, final int length); + + short getShort(int position); + + short getShort(); + + default short[] getShortArray() { // NOPMD by rstein + return getShortArray(null, 0); + } + + default short[] getShortArray(final short[] dst) { // NOPMD by rstein + return getShortArray(dst, dst == null ? -1 : dst.length); + } + + short[] getShortArray(final short[] dst, final int length); + + String getString(int position); + + String getString(); + + default String[] getStringArray() { + return getStringArray(null, 0); + } + + default String[] getStringArray(final String[] dst) { + return getStringArray(dst, dst == null ? -1 : dst.length); + } + + String[] getStringArray(final String[] dst, final int length); + + String getStringISO8859(); + + /** + * @return {@code true} the ISO-8859-1 character encoding is being enforced for data fields (better performance), otherwise UTF-8 is being used (more generic encoding) + */ + boolean isEnforceSimpleStringEncoding(); + + /** + * @param state {@code true} the ISO-8859-1 character encoding is being enforced for data fields (better performance), otherwise UTF-8 is being used (more generic encoding) + */ + void setEnforceSimpleStringEncoding(boolean state); + + void putBoolean(int position, boolean value); + + void putBoolean(boolean value); + + void putBooleanArray(final boolean[] src, final int n); + + void putByte(int position, byte value); + + void putByte(final byte b); + + void putByteArray(final byte[] src, final int n); + + void putChar(int position, char value); + + void putChar(char value); + + void putCharArray(final char[] src, final int n); + + void putDouble(int position, double value); + + void putDouble(double value); + + void putDoubleArray(final double[] src, final int n); + + void putFloat(int position, float value); + + void putFloat(float value); + + void putFloatArray(final float[] src, final int n); + + void putInt(int position, int value); + + void putInt(int value); + + void putIntArray(final int[] src, final int n); + + void putLong(int position, long value); + + void putLong(long value); + + void putLongArray(final long[] src, final int n); + + void putShort(int position, short value); + + void putShort(short value); + + void putShortArray(final short[] src, final int n); + + void putString(int position, String value); + + void putString(String string); + + void putStringArray(final String[] src, final int n); + + void putStringISO8859(String string); +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/IoBufferHeader.java b/serialiser/src/main/java/io/opencmw/serialiser/IoBufferHeader.java new file mode 100644 index 00000000..b7ab5ca5 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/IoBufferHeader.java @@ -0,0 +1,160 @@ +package io.opencmw.serialiser; + +import java.util.concurrent.locks.ReadWriteLock; + +/** + * Interface definition in line with the jdk Buffer abstract class. This definition is needed to allow to redirect and + * allow for different buffer implementations. + *

+ * A buffer is a linear, finite sequence of elements of a specific primitive type. Aside from its content, the essential + * properties of a buffer are its capacity, limit, and position: + *

+ *
+ *

+ * A buffer's capacity is the number of elements it contains. The capacity of a buffer is never negative and + * never changes. + *

+ *

+ * A buffer's limit is the index of the first element that should not be read or written. A buffer's limit is + * never negative and is never greater than its capacity. + *

+ *

+ * A buffer's position is the index of the next element to be read or written. A buffer's position is never + * negative and is never greater than its limit. + *

+ *
+ *

+ * The following invariant holds for the mark, position, limit, and capacity values:

{@code 0} {@code <=} + * position {@code <=} limit {@code <=} capacity
+ * + * @author rstein + */ +@SuppressWarnings("PMD.TooManyMethods") // NOPMD - these are short-hand convenience methods +public interface IoBufferHeader { + /** + * @return the capacity of this buffer + */ + int capacity(); + + /** + * Clears this buffer. The position is set to zero amd the limit is set to the capacity. + *

+ * Invoke this method before using a sequence of channel-read or put operations to fill this buffer. For + * example:

+ * + * 
+     * buf.clear(); // Prepare buffer for reading
+     * in.read(buf); // Read data
+     *
+ * + *
+ *

+ * This method does not actually erase the data in the buffer, but it is named as if it did because it will most + * often be used in situations in which that might as well be the case. + *

+ */ + void clear(); + + void ensureAdditionalCapacity(final int capacity); + + void ensureCapacity(final int capacity); + + /** + * Flips this buffer. The limit is set to the current position and then + * the position is set to zero. If the mark is defined then it is + * discarded. + * + *

After a sequence of channel-read or put operations, invoke + * this method to prepare for a sequence of channel-write or relative + * get operations. For example: + * + * 

+     * buf.put(magic);    // Prepend header
+     * in.read(buf);      // Read data into rest of buffer
+     * buf.flip();        // Flip buffer
+     * out.write(buf);    // Write header + data to channel
+ */ + void flip(); + + /** + * Forces buffer to contain the given number of entries, preserving just a part of the array. + * + * @param length the new minimum length for this array. + * @param preserve the number of elements of the old buffer that shall be preserved in case a new allocation is + * necessary. + */ + void forceCapacity(final int length, final int preserve); + + /** + * @return {@code true} if, and only if, there is at least one element remaining in this buffer + */ + boolean hasRemaining(); + + /** + * @return {@code true} if, and only if, this buffer is read-only + */ + boolean isReadOnly(); + + /** + * @return the limit of this buffer + */ + int limit(); + + /** + * Sets this buffer's limit. If the position is larger than the new limit then it is set to the new limit. If the + * mark is defined and larger than the new limit then it is discarded. + * + * @param newLimit the new limit value; must be non-negative and no larger than this buffer's capacity + */ + void limit(final int newLimit); + + /** + * For efficiency/performance reasons the buffer implementation is not required to safe-guard each put/get method + * independently. Thus the user-code should acquire the given lock around a set of put/get appropriately. + * + * @return the read-write lock + */ + ReadWriteLock lock(); + + /** + * @return the position of this buffer + */ + int position(); + + /** + * Sets this buffer's position. If the mark is defined and larger than the new position then it is discarded. + * + * @param newPosition the new position value; must be non-negative and no larger than the current limit + */ + void position(final int newPosition); + + /** + * @return the number of elements remaining in this buffer + */ + int remaining(); + + /** + * resets the buffer read/write position to zero + */ + void reset(); + + /** + * Trims the internal buffer array so that the capacity is equal to the size. + * + * @see java.util.ArrayList#trimToSize() + */ + void trim(); + + /** + * Trims the internal buffer array if it is too large. If the current array length is smaller than or equal to + * {@code n}, this method does nothing. Otherwise, it trims the array length to the maximum between + * {@code requestedCapacity} and {@link #capacity()}. + *

+ * This method is useful when reusing FastBuffers. {@linkplain #reset() Clearing a list} leaves the array length + * untouched. If you are reusing a list many times, you can call this method with a typical size to avoid keeping + * around a very large array just because of a few large transient lists. + * + * @param requestedCapacity the threshold for the trimming. + */ + void trim(final int requestedCapacity); +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/IoClassSerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/IoClassSerialiser.java new file mode 100644 index 00000000..555aee36 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/IoClassSerialiser.java @@ -0,0 +1,627 @@ +package io.opencmw.serialiser; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ClassFieldDescription; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.JsonSerialiser; +import io.opencmw.serialiser.spi.WireDataFieldDescription; +import io.opencmw.serialiser.spi.iobuffer.FieldBoxedValueArrayHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldBoxedValueHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldCollectionsHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldDataSetHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldMapHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldMultiArrayHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldPrimitiveValueHelper; +import io.opencmw.serialiser.spi.iobuffer.FieldPrimitveValueArrayHelper; +import io.opencmw.serialiser.utils.ClassUtils; + +import de.gsi.dataset.utils.ByteArrayCache; + +/** + * reference implementation for streaming arbitrary object classes to and from a IoSerialiser- and IoBuffer-based buffers + * + * @author rstein + */ +@SuppressWarnings({ "PMD.TooManyMethods", "PMD.ExcessiveImports", "PMD.NPathComplexity" }) +public class IoClassSerialiser { + private static final Logger LOGGER = LoggerFactory.getLogger(IoClassSerialiser.class); + public static final String UNCHECKED_CAST_SUPPRESSION = "unchecked"; + private static final Map> CLASS_CONSTRUCTOR_MAP = new ConcurrentHashMap<>(); + protected final List ioSerialisers = new ArrayList<>(); + private final Map>> classMap = new ConcurrentHashMap<>(); + private final Map cachedFieldMatch = new ConcurrentHashMap<>(); + protected IoSerialiser matchedIoSerialiser; + protected IoBuffer dataBuffer; + protected Consumer startMarkerFunction; + protected Consumer endMarkerFunction; + private boolean autoMatchSerialiser = true; + private boolean useCustomJsonSerialiser; + + /** + * Initialises new IoBuffer-backed object serialiser + * + * @param ioBuffer the backing IoBuffer (see e.g. {@link IoBuffer} + * @param ioSerialiserTypeClass optional IoSerialiser type class this IoClassSerialiser should start with + * (see also e.g. {@link BinarySerialiser}, + * {@link CmwLightSerialiser}, or + * {@link JsonSerialiser} + */ + @SafeVarargs + public IoClassSerialiser(final IoBuffer ioBuffer, final Class... ioSerialiserTypeClass) { + dataBuffer = ioBuffer; + // add default IoSerialiser Implementations + ioSerialisers.add(new BinarySerialiser(dataBuffer)); + ioSerialisers.add(new JsonSerialiser(dataBuffer)); + ioSerialisers.add(new CmwLightSerialiser(dataBuffer)); + if (ioSerialiserTypeClass.length > 0) { + setMatchedIoSerialiser(ioSerialiserTypeClass[0]); // NOPMD + } else { + setMatchedIoSerialiser(ioSerialisers.get(0)); // NOPMD + } + + // register primitive and boxed data type handlers + FieldPrimitiveValueHelper.register(this); + FieldPrimitveValueArrayHelper.register(this); + FieldBoxedValueHelper.register(this); + FieldBoxedValueArrayHelper.register(this); + FieldCollectionsHelper.register(this); + + // Enum serialiser mapper to IoBuffer + addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getEnum((Enum) field.getField().get(obj))), // reader + (io, obj, field) -> io.getEnum((Enum) (field == null ? obj : field.getField().get(obj))), // return + (io, obj, field) -> io.put(field, (Enum) field.getField().get(obj)), // writer + Enum.class)); + + FieldMapHelper.register(this); + FieldDataSetHelper.register(this); + // MultiArray handlers + FieldMultiArrayHelper.register(this); + } + + public void addClassDefinition(FieldSerialiser serialiser) { + if (serialiser == null) { + throw new IllegalArgumentException("serialiser must not be null"); + } + if (serialiser.getClassPrototype() == null) { + throw new IllegalArgumentException("clazz must not be null"); + } + if (serialiser.getGenericsPrototypes() == null) { + throw new IllegalArgumentException("types must not be null"); + } + synchronized (knownClasses()) { + final List> list = knownClasses().computeIfAbsent(serialiser.getClassPrototype(), key -> new ArrayList<>()); + + if (list.isEmpty() || !list.contains(serialiser)) { + list.add(serialiser); + } + } + } + + /** + * if enabled ({@link #isAutoMatchSerialiser()} then set matching serialiser + */ + public void autoUpdateSerialiser() { + ioSerialisers.forEach(s -> s.setBuffer(dataBuffer)); + if (!isAutoMatchSerialiser()) { + return; + } + final int originalPosition = dataBuffer.position(); + for (IoSerialiser ioSerialiser : ioSerialisers) { + try { + ioSerialiser.checkHeaderInfo(); + this.setMatchedIoSerialiser(ioSerialiser); + LOGGER.atTrace().addArgument(matchedIoSerialiser).addArgument(matchedIoSerialiser.getBuffer().capacity()).log("set autoUpdateSerialiser() to {} - buffer capacity = {}"); + dataBuffer.position(originalPosition); + return; + } catch (Throwable e) { // NOPMD NOSONAR expected failures for protocol mismatch + LOGGER.atTrace().setCause(e).addArgument(ioSerialiser).log("could not match IoSerialiser '{}'"); + } + dataBuffer.position(originalPosition); + } + } + + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + public FieldSerialiser cacheFindFieldSerialiser(Type clazz, List classGenericArguments) { + // odd construction is needed since 'computeIfAbsent' cannot place 'null' element into the Map and since 'null' has a double interpretation of + // a) a non-initialiser map value + // b) a class for which no custom serialiser exist + return (FieldSerialiser) cachedFieldMatch.computeIfAbsent(new FieldSerialiserKey(clazz, classGenericArguments), key -> new FieldSerialiserValue(findFieldSerialiser(clazz, classGenericArguments))).get(); + } + + public T deserialiseObject(WireDataFieldDescription fieldRoot, final T obj) { + autoUpdateSerialiser(); + final int startPosition = matchedIoSerialiser.getBuffer().position(); + + // match field header with class field description + final ClassFieldDescription clazz = ClassUtils.getFieldDescription(obj.getClass()); + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser existingSerialiser = (FieldSerialiser) clazz.getFieldSerialiser(); + final FieldSerialiser fieldSerialiser = existingSerialiser == null ? cacheFindFieldSerialiser(clazz.getType(), clazz.getActualTypeArguments()) : existingSerialiser; + + if (clazz.getFieldSerialiser() == null && fieldSerialiser != null) { + clazz.setFieldSerialiser(fieldSerialiser); + } + + matchedIoSerialiser.getBuffer().position(startPosition); + + if (fieldSerialiser != null) { + // return new object + final FieldDescription rawObjectFieldDescription = fieldRoot.getChildren().get(0).getChildren().get(0); + matchedIoSerialiser.getBuffer().position(rawObjectFieldDescription.getDataStartPosition()); + if (rawObjectFieldDescription.getDataType() == DataType.OTHER) { + return matchedIoSerialiser.getCustomData(fieldSerialiser); + } + return fieldSerialiser.getReturnObjectFunction().apply(matchedIoSerialiser, obj, clazz); + } + // deserialise into object + if (!fieldRoot.getChildren().isEmpty() && !fieldRoot.getChildren().get(0).getFieldName().isEmpty()) { + for (final FieldDescription child : fieldRoot.getChildren()) { + deserialise(obj, obj.getClass(), child, clazz, 0); + } + return obj; + } + + // class reference is not known by name (ie. was empty) parse directly dependent children + final List fieldRootChildren = fieldRoot.getChildren().get(0).getChildren(); + for (final FieldDescription fieldDescription : fieldRootChildren) { + final ClassFieldDescription subFieldDescription = (ClassFieldDescription) clazz.findChildField(fieldDescription.getFieldNameHashCode(), fieldDescription.getFieldName()); + + if (subFieldDescription != null) { + deserialise(obj, obj.getClass(), fieldDescription, subFieldDescription, 1); + } + } + return obj; + } + + public T deserialiseObject(final Class clazz) { + try { + final Constructor constructor = clazz.getDeclaredConstructor(); + constructor.setAccessible(true); + T obj = constructor.newInstance(); + return deserialiseObject(obj); + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException("no public constructor for class " + clazz.getCanonicalName(), e); + } + } + + public T deserialiseObject(final T obj) { + if (obj == null) { + throw new IllegalArgumentException("obj must not be null (yet)"); + } + autoUpdateSerialiser(); + // try to match buffer + if (matchedIoSerialiser instanceof JsonSerialiser) { + return ((JsonSerialiser) matchedIoSerialiser).deserialiseObject(obj); + } + final WireDataFieldDescription fieldRoot = parseWireFormat(); + return deserialiseObject(fieldRoot, obj); + } + + public void finaliseBuffer(ByteArrayCache arrayCache) { + try { + if (arrayCache == null) { + ByteArrayCache.getInstance().add(dataBuffer.elements()); + } else { + arrayCache.add(dataBuffer.elements()); + } + // return buffer to cache + dataBuffer = null; // NOPMD on purpose + for (IoSerialiser serialiser : ioSerialisers) { + serialiser.setBuffer(null); + } + } catch (Exception e) { // NOPMD + // do nothing + } + } + + /** + * find FieldSerialiser for known class, interface and corresponding generics + * @param type the class or interface + * @param classGenericArguments optional generics arguments + * @return FieldSerialiser matching the base class/interface and generics arguments + * @param The type of the Object to (de)serialise + */ + @SuppressWarnings({ UNCHECKED_CAST_SUPPRESSION }) + public FieldSerialiser findFieldSerialiser(Type type, List classGenericArguments) { + final Class clazz = ClassUtils.getRawType(type); + if (clazz == null) { + throw new IllegalArgumentException("clazz must not be null"); + } + final List> directClassMatchList = classMap.get(type); + if (directClassMatchList != null && !directClassMatchList.isEmpty()) { + if (directClassMatchList.size() == 1 || classGenericArguments == null || classGenericArguments.isEmpty()) { + return (FieldSerialiser) directClassMatchList.get(0); + } + // more than one possible serialiser implementation + for (final FieldSerialiser entry : directClassMatchList) { + if (checkClassCompatibility(classGenericArguments, entry.getGenericsPrototypes())) { + return (FieldSerialiser) entry; + } + } + // found FieldSerialiser entry but not matching required generic types + } + + // did not find FieldSerialiser entry by specific class -> search for assignable interface definitions + + final List> potentialMatchingKeys = new ArrayList<>(10); + for (Type key : knownClasses().keySet()) { + final Class testClass = ClassUtils.getRawType(key); + if (testClass.isAssignableFrom(clazz)) { + potentialMatchingKeys.add(testClass); + } + } + if (potentialMatchingKeys.isEmpty()) { + // did not find any matching clazz/interface FieldSerialiser entries + return null; + } + + final List> interfaceMatchList = new ArrayList<>(10); + for (Class testClass : potentialMatchingKeys) { + final List> fieldSerialisers = knownClasses().get(testClass); + if (fieldSerialisers.isEmpty()) { + continue; + } + interfaceMatchList.addAll(fieldSerialisers); + } + if (interfaceMatchList.size() == 1 || classGenericArguments == null || classGenericArguments.isEmpty()) { + // found single match FieldSerialiser entry type w/o specific generics requirements + return (FieldSerialiser) interfaceMatchList.get(0); + } + + // more than one possible serialiser implementation + for (final FieldSerialiser entry : interfaceMatchList) { + if (checkClassCompatibility(classGenericArguments, entry.getGenericsPrototypes())) { + // found generics matching or assignable entry + return (FieldSerialiser) entry; + } + } + // could not match with generics arguments + + // find generic serialiser entry w/o generics parameter requirements + return (FieldSerialiser) interfaceMatchList.stream().filter(entry -> entry.getGenericsPrototypes().isEmpty()).findFirst().orElse(null); + } + + public IoBuffer getDataBuffer() { + return dataBuffer; + } + + public IoSerialiser getMatchedIoSerialiser() { + return matchedIoSerialiser; + } + + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + public final BiFunction> getSerialiserLookupFunction() { + return (primaryType, secondaryType) -> { + if (primaryType == null) { + throw new IllegalArgumentException("no serialiser implementation found for classType = " + null); + } + return (FieldSerialiser) cacheFindFieldSerialiser(ClassUtils.getRawType(primaryType), secondaryType == null ? Collections.emptyList() : Arrays.asList(secondaryType)); + }; + } + + public boolean isAutoMatchSerialiser() { + return autoMatchSerialiser; + } + + public boolean isUseCustomJsonSerialiser() { + return useCustomJsonSerialiser; + } + + public Map>> knownClasses() { + return classMap; + } + + public WireDataFieldDescription parseWireFormat() { + autoUpdateSerialiser(); + final int startPosition = matchedIoSerialiser.getBuffer().position(); + matchedIoSerialiser.getBuffer().position(startPosition); + return matchedIoSerialiser.parseIoStream(true); + } + + public void serialiseObject(final Object rootObj, final ClassFieldDescription classField, final int recursionDepth) { + final FieldSerialiser existingSerialiser = classField.getFieldSerialiser(); + final FieldSerialiser fieldSerialiser = existingSerialiser == null ? cacheFindFieldSerialiser(classField.getType(), classField.getActualTypeArguments()) : existingSerialiser; + + if (fieldSerialiser != null && recursionDepth != 0) { + if (existingSerialiser == null) { + classField.setFieldSerialiser(fieldSerialiser); + } + // write field header + if (classField.getDataType() == DataType.OTHER) { + final WireDataFieldDescription header = matchedIoSerialiser.putFieldHeader(classField.getFieldName(), classField.getDataType()); + fieldSerialiser.getWriterFunction().accept(matchedIoSerialiser, rootObj, classField); + matchedIoSerialiser.updateDataEndMarker(header); + } else { + fieldSerialiser.getWriterFunction().accept(matchedIoSerialiser, rootObj, classField); + } + return; + } + // cannot serialise field check whether this is a container class and contains serialisable children + + if (classField.getChildren().isEmpty()) { + // no further children + return; + } + + // dive into it's children + if (recursionDepth != 0 && startMarkerFunction != null) { + startMarkerFunction.accept(classField); + } + + final Object newRoot = classField.getField() == null ? rootObj : classField.getField().get(rootObj); + for (final FieldDescription fieldDescription : classField.getChildren()) { + ClassFieldDescription field = (ClassFieldDescription) fieldDescription; + + if (!field.isPrimitive()) { + final Object reference = field.getField().get(newRoot); + if (!field.isPrimitive() && reference == null) { + // only follow and serialise non-null references of sub-classes + continue; + } + } + serialiseObject(newRoot, field, recursionDepth + 1); + } + + if (recursionDepth != 0 && endMarkerFunction != null) { + endMarkerFunction.accept(classField); + } + } + + public void serialiseObject(final Object obj) { + if (matchedIoSerialiser instanceof JsonSerialiser && useCustomJsonSerialiser) { + ((JsonSerialiser) matchedIoSerialiser).serialiseObject(obj); + return; + } + if (obj == null) { + // serialise null object + matchedIoSerialiser.putHeaderInfo(); + final String dataEndMarkerName = "OBJ_ROOT_END"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(matchedIoSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + matchedIoSerialiser.putEndMarker(dataEndMarker); + return; + } + + final ClassFieldDescription classField = ClassUtils.getFieldDescription(obj.getClass()); + final FieldSerialiser existingSerialiser = classField.getFieldSerialiser(); + @SuppressWarnings("rawtypes") + final FieldSerialiser fieldSerialiser = existingSerialiser == null ? cacheFindFieldSerialiser(classField.getType(), classField.getActualTypeArguments()) : existingSerialiser; + + if (fieldSerialiser == null) { + matchedIoSerialiser.putHeaderInfo(classField); + serialiseObject(obj, classField, 0); + matchedIoSerialiser.putEndMarker(classField); + } else { + if (existingSerialiser == null) { + classField.setFieldSerialiser(fieldSerialiser); + } + matchedIoSerialiser.putHeaderInfo(); + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + FieldSerialiser castFieldSerialiser = fieldSerialiser; + matchedIoSerialiser.putCustomData(classField, obj, obj.getClass(), castFieldSerialiser); + final String dataEndMarkerName = "OBJ_ROOT_END"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(matchedIoSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + matchedIoSerialiser.putEndMarker(dataEndMarker); + } + } + + public void setAutoMatchSerialiser(final boolean autoMatchSerialiser) { + this.autoMatchSerialiser = autoMatchSerialiser; + } + + public void setDataBuffer(final IoBuffer dataBuffer) { + this.dataBuffer = dataBuffer; + } + + public void setMatchedIoSerialiser(final Class serialiserTemplate) { + if (serialiserTemplate == null) { + throw new IllegalArgumentException("serialiserTemplate must not be null"); + } + + for (IoSerialiser ioSerialiser : ioSerialisers) { + if (ioSerialiser.getClass().equals(serialiserTemplate)) { + setMatchedIoSerialiser(ioSerialiser); + return; + } + } + throw new IllegalArgumentException("IoSerialiser '" + serialiserTemplate.getCanonicalName() + "' not registered with this = " + this); + } + + public void setUseCustomJsonSerialiser(final boolean useCustomJsonSerialiser) { + this.useCustomJsonSerialiser = useCustomJsonSerialiser; + } + + protected boolean checkClassCompatibility(final List ref1, final List ref2) { + if (ref1.size() != ref2.size()) { + return false; + } + if (ref1.isEmpty()) { + return true; + } + + for (int i = 0; i < ref1.size(); i++) { + final Class class1 = ClassUtils.getRawType(ref1.get(i)); + final Class class2 = ClassUtils.getRawType(ref2.get(i)); + if (!class1.equals(class2) && !(class2.isAssignableFrom(class1))) { + return false; + } + } + + return true; + } + + protected void deserialise(final Object obj, Class clazz, final FieldDescription fieldRoot, final ClassFieldDescription classField, final int recursionDepth) { + assert obj != null; + assert clazz != null; + @SuppressWarnings("rawtypes") + final FieldSerialiser existingSerialiser = classField.getFieldSerialiser(); + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser fieldSerialiser = existingSerialiser == null ? cacheFindFieldSerialiser(classField.getType(), classField.getActualTypeArguments()) : existingSerialiser; + + if (fieldSerialiser != null) { + if (existingSerialiser == null) { + classField.setFieldSerialiser(fieldSerialiser); + } + matchedIoSerialiser.getBuffer().position(fieldRoot.getDataStartPosition()); + classField.getFieldSerialiser().getReaderFunction().accept(matchedIoSerialiser, obj, classField); + return; + } + + if (fieldRoot.getFieldNameHashCode() != classField.getFieldNameHashCode() /*|| !fieldRoot.getFieldName().equals(classField.getFieldName())*/) { + // did not find matching (sub-)field in class + if (fieldRoot.getChildren().isEmpty()) { + return; + } + // check for potential inner fields + for (final FieldDescription fieldDescription : fieldRoot.getChildren()) { + final ClassFieldDescription subFieldDescription = (ClassFieldDescription) classField.findChildField(fieldDescription.getFieldNameHashCode(), fieldDescription.getFieldName()); + + if (subFieldDescription != null) { + deserialise(obj, obj.getClass(), fieldDescription, subFieldDescription, recursionDepth + 1); + } + } + return; + } + + final Class fieldClass = ClassUtils.getRawType(classField.getType()); + if (classField.isFinal() && !fieldClass.isInterface()) { + // cannot set final variables + LOGGER.atWarn().addArgument(classField.getParent()).addArgument(classField.getFieldName()).log("cannot (read: better should not) set final field '{}-{}'"); + return; + } + + final Object ref = classField.getField() == null ? obj : classField.getField().get(obj); + final Object subRef; + if (ref == null) { + subRef = classField.allocateMemberClassField(obj); + } else { + subRef = ref; + } + + // no specific deserialiser present check for potential inner fields + for (final FieldDescription fieldDescription : fieldRoot.getChildren()) { + final ClassFieldDescription subFieldDescription = (ClassFieldDescription) classField.findChildField(fieldDescription.getFieldNameHashCode(), fieldDescription.getFieldName()); + + if (subFieldDescription != null) { + deserialise(subRef, subRef.getClass(), fieldDescription, subFieldDescription, recursionDepth + 1); + } + } + } + + private void setMatchedIoSerialiser(final IoSerialiser matchedIoSerialiser) { + this.matchedIoSerialiser = matchedIoSerialiser; + this.matchedIoSerialiser.setBuffer(dataBuffer); + this.matchedIoSerialiser.setFieldSerialiserLookupFunction(getSerialiserLookupFunction()); + assert this.matchedIoSerialiser.getBuffer() == dataBuffer; + startMarkerFunction = this.matchedIoSerialiser::putStartMarker; + endMarkerFunction = this.matchedIoSerialiser::putEndMarker; + LOGGER.atTrace().addArgument(matchedIoSerialiser).log("setMatchedIoSerialiser to {}"); + } + + public static int computeHashCode(final Class classPrototype, List classGenericArguments) { + final int prime = 31; + int result = 1; + result = prime * result + ((classPrototype == null) ? 0 : classPrototype.getName().hashCode()); + if (classGenericArguments == null || classGenericArguments.isEmpty()) { + return result; + } + for (final Type arg : classGenericArguments) { + result = prime * result + ((arg == null) ? 0 : arg.getTypeName().hashCode()); + } + + return result; + } + + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + public static Constructor getClassConstructorByName(final String name, Class... parameterTypes) { + return CLASS_CONSTRUCTOR_MAP.computeIfAbsent(name, key -> { + try { + return (Constructor) ClassUtils.getClassByName(key) + .getDeclaredConstructor(parameterTypes); + } catch (SecurityException | NoSuchMethodException e) { + LOGGER.atError().setCause(e).addArgument(Arrays.toString(parameterTypes)).addArgument(name).log("exception while getting constructor{} for class {}"); + return null; + } + }); + } + + public static String[] getClassNames(List> classGenericArguments) { + if (classGenericArguments == null) { + return new String[0]; + } + final String[] argStrings = new String[classGenericArguments.size()]; + for (int i = 0; i < argStrings.length; i++) { + argStrings[i] = classGenericArguments.get(i).getName(); + } + return argStrings; + } + + public static String getGenericFieldSimpleTypeString(List classArguments) { + if (classArguments == null || classArguments.isEmpty()) { + return ""; + } + return classArguments.stream().map(Type::getTypeName).collect(Collectors.joining(", ", "<", ">")); + } + + private static class FieldSerialiserKey { + private final Type clazz; + private final List classGenericArguments; + + private FieldSerialiserKey(Type clazz, List classGenericArguments) { + this.clazz = clazz; + this.classGenericArguments = classGenericArguments; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final FieldSerialiserKey that = (FieldSerialiserKey) o; + return clazz.equals(that.clazz) && classGenericArguments.equals(that.classGenericArguments); + } + + @Override + public int hashCode() { + return Objects.hash(clazz, classGenericArguments); + } + + @Override + public String toString() { + return "FieldSerialiserKey{" + + "clazz=" + clazz + ", classGenericArguments=" + classGenericArguments + '}'; + } + } + + private static class FieldSerialiserValue { + private final FieldSerialiser fieldSerialiser; + + private FieldSerialiserValue(FieldSerialiser fieldSerialiser) { + this.fieldSerialiser = fieldSerialiser; + } + + private FieldSerialiser get() { + return fieldSerialiser; + } + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/IoSerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/IoSerialiser.java new file mode 100644 index 00000000..6c1d330d --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/IoSerialiser.java @@ -0,0 +1,382 @@ +package io.opencmw.serialiser; + +import java.lang.reflect.Type; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.function.BiFunction; + +import io.opencmw.serialiser.spi.ProtocolInfo; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +@SuppressWarnings({ "PMD.TooManyMethods", "PMD.ExcessivePublicCount", "PMD.AvoidUsingShortType" }) // unavoidable since Java does not support templates (issue: primitive types) +public interface IoSerialiser { + /** + * Reads and checks protocol header information. + * @return ProtocolInfo info Object (extends FieldHeader) + * @throws IllegalStateException in case the format is incompatible with this serialiser + */ + ProtocolInfo checkHeaderInfo(); + + void setQueryFieldName(String fieldName, final int dataStartPosition); + + int[] getArraySizeDescriptor(); + + boolean getBoolean(); // NOPMD by rstein + + default boolean[] getBooleanArray() { + return getBooleanArray(null, 0); + } + + default boolean[] getBooleanArray(final boolean[] dst) { + return getBooleanArray(dst, dst == null ? -1 : dst.length); + } + + boolean[] getBooleanArray(final boolean[] dst, final int length); + + IoBuffer getBuffer(); + + void setBuffer(IoBuffer buffer); + + byte getByte(); + + default byte[] getByteArray() { + return getByteArray(null, 0); + } + + default byte[] getByteArray(final byte[] dst) { + return getByteArray(dst, dst == null ? -1 : dst.length); + } + + byte[] getByteArray(final byte[] dst, final int length); + + char getChar(); + + default char[] getCharArray() { + return getCharArray(null, 0); + } + + default char[] getCharArray(final char[] dst) { + return getCharArray(dst, dst == null ? -1 : dst.length); + } + + char[] getCharArray(final char[] dst, final int length); + + Collection getCollection(Collection collection); + + E getCustomData(FieldSerialiser serialiser); + + double getDouble(); + + default double[] getDoubleArray() { + return getDoubleArray(null, 0); + } + + default double[] getDoubleArray(final double[] dst) { + return getDoubleArray(dst, dst == null ? -1 : dst.length); + } + + double[] getDoubleArray(final double[] dst, final int length); + + > Enum getEnum(Enum enumeration); + + String getEnumTypeList(); + + WireDataFieldDescription getFieldHeader(); + + float getFloat(); + + default float[] getFloatArray() { + return getFloatArray(null, 0); + } + + default float[] getFloatArray(final float[] dst) { + return getFloatArray(dst, dst == null ? -1 : dst.length); + } + + float[] getFloatArray(final float[] dst, final int length); + + int getInt(); + + default int[] getIntArray() { + return getIntArray(null, 0); + } + + default int[] getIntArray(final int[] dst) { + return getIntArray(dst, dst == null ? -1 : dst.length); + } + + int[] getIntArray(final int[] dst, final int length); + + List getList(List collection); + + long getLong(); + + default long[] getLongArray() { + return getLongArray(null, 0); + } + + default long[] getLongArray(final long[] dst) { + return getLongArray(dst, dst == null ? -1 : dst.length); + } + + long[] getLongArray(final long[] dst, final int length); + + Map getMap(Map map); + + Queue getQueue(Queue collection); + + Set getSet(Set collection); + + short getShort(); // NOPMD by rstein + + default short[] getShortArray() { // NOPMD by rstein + return getShortArray(null, 0); + } + + default short[] getShortArray(final short[] dst) { // NOPMD by rstein + return getShortArray(dst, dst == null ? -1 : dst.length); + } + + short[] getShortArray(final short[] dst, final int length); // NOPMD by rstein + + String getString(); + + default String[] getStringArray() { + return getStringArray(null, 0); + } + + default String[] getStringArray(final String[] dst) { + return getStringArray(dst, dst == null ? -1 : dst.length); + } + + String[] getStringArray(final String[] dst, final int length); + + String getStringISO8859(); + + boolean isPutFieldMetaData(); + + void setPutFieldMetaData(boolean putFieldMetaData); + + WireDataFieldDescription parseIoStream(boolean readHeader); + + void put(FieldDescription fieldDescription, Collection collection, Type valueType); + + void put(FieldDescription fieldDescription, Enum enumeration); + + void put(FieldDescription fieldDescription, Map map, Type keyType, Type valueType); + + void put(String fieldName, Collection collection, Type valueType); + + void put(String fieldName, Enum enumeration); + + void put(String fieldName, Map map, Type keyType, Type valueType); + + default void put(FieldDescription fieldDescription, final boolean[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final byte[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final char[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final double[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final float[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final int[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final long[] src) { + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final short[] src) { // NOPMD + put(fieldDescription, src, -1); + } + + default void put(FieldDescription fieldDescription, final String[] src) { + put(fieldDescription, src, -1); + } + + void put(FieldDescription fieldDescription, boolean value); + + void put(FieldDescription fieldDescription, boolean[] values, int n); + + void put(FieldDescription fieldDescription, boolean[] values, int[] dims); + + void put(FieldDescription fieldDescription, byte value); + + void put(FieldDescription fieldDescription, byte[] values, int n); + + void put(FieldDescription fieldDescription, byte[] values, int[] dims); + + void put(FieldDescription fieldDescription, char value); + + void put(FieldDescription fieldDescription, char[] values, int n); + + void put(FieldDescription fieldDescription, char[] values, int[] dims); + + void put(FieldDescription fieldDescription, double value); + + void put(FieldDescription fieldDescription, double[] values, int n); + + void put(FieldDescription fieldDescription, double[] values, int[] dims); + + void put(FieldDescription fieldDescription, float value); + + void put(FieldDescription fieldDescription, float[] values, int n); + + void put(FieldDescription fieldDescription, float[] values, int[] dims); + + void put(FieldDescription fieldDescription, int value); + + void put(FieldDescription fieldDescription, int[] values, int n); + + void put(FieldDescription fieldDescription, int[] values, int[] dims); + + void put(FieldDescription fieldDescription, long value); + + void put(FieldDescription fieldDescription, long[] values, int n); + + void put(FieldDescription fieldDescription, long[] values, int[] dims); + + void put(FieldDescription fieldDescription, short value); + + void put(FieldDescription fieldDescription, short[] values, int n); + + void put(FieldDescription fieldDescription, short[] values, int[] dims); + + void put(FieldDescription fieldDescription, String string); + + void put(FieldDescription fieldDescription, String[] values, int n); + + void put(FieldDescription fieldDescription, String[] values, int[] dims); + + default void put(String fieldName, final boolean[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final byte[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final char[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final double[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final float[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final int[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final long[] src) { + put(fieldName, src, -1); + } + + default void put(String fieldName, final short[] src) { // NOPMD + put(fieldName, src, -1); + } + + default void put(String fieldName, final String[] src) { + put(fieldName, src, -1); + } + + void put(String fieldName, boolean value); + + void put(String fieldName, boolean[] values, int n); + + void put(String fieldName, boolean[] values, int[] dims); + + void put(String fieldName, byte value); + + void put(String fieldName, byte[] values, int n); + + void put(String fieldName, byte[] values, int[] dims); + + void put(String fieldName, char value); + + void put(String fieldName, char[] values, int n); + + void put(String fieldName, char[] values, int[] dims); + + void put(String fieldName, double value); + + void put(String fieldName, double[] values, int n); + + void put(String fieldName, double[] values, int[] dims); + + void put(String fieldName, float value); + + void put(String fieldName, float[] values, int n); + + void put(String fieldName, float[] values, int[] dims); + + void put(String fieldName, int value); + + void put(String fieldName, int[] values, int n); + + void put(String fieldName, int[] values, int[] dims); + + void put(String fieldName, long value); + + void put(String fieldName, long[] values, int n); + + void put(String fieldName, long[] values, int[] dims); + + void put(String fieldName, short value); + + void put(String fieldName, short[] values, int n); + + void put(String fieldName, short[] values, int[] dims); + + void put(String fieldName, String string); + + void put(String fieldName, String[] values, int n); + + void put(String fieldName, String[] values, int[] dims); + + int putArraySizeDescriptor(int n); + + int putArraySizeDescriptor(int[] dims); + + WireDataFieldDescription putCustomData(FieldDescription fieldDescription, E obj, Class type, FieldSerialiser serialiser); + + void putEndMarker(FieldDescription fieldDescription); + + WireDataFieldDescription putFieldHeader(FieldDescription fieldDescription); + + WireDataFieldDescription putFieldHeader(String fieldName, DataType dataType); + + /** + * Adds header and version information + * @param field optional FieldDescription (ie. to allow to attach MetaData to the start/stop marker) + */ + void putHeaderInfo(FieldDescription... field); + + void putStartMarker(FieldDescription fieldDescription); + + void updateDataEndMarker(WireDataFieldDescription fieldHeader); + + void setFieldSerialiserLookupFunction(BiFunction> serialiserLookupFunction); + + BiFunction> getSerialiserLookupFunction(); +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/annotations/Description.java b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Description.java new file mode 100644 index 00000000..76c81b8d --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Description.java @@ -0,0 +1,12 @@ +package io.opencmw.serialiser.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.TYPE }) +public @interface Description { + String value() default ""; +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/annotations/Direction.java b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Direction.java new file mode 100644 index 00000000..815da5d7 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Direction.java @@ -0,0 +1,12 @@ +package io.opencmw.serialiser.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.TYPE }) +public @interface Direction { + String value() default ""; +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/annotations/Groups.java b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Groups.java new file mode 100644 index 00000000..585c4e2c --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Groups.java @@ -0,0 +1,12 @@ +package io.opencmw.serialiser.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.TYPE }) +public @interface Groups { + String[] value() default ""; +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/annotations/MetaInfo.java b/serialiser/src/main/java/io/opencmw/serialiser/annotations/MetaInfo.java new file mode 100644 index 00000000..8659a6f0 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/annotations/MetaInfo.java @@ -0,0 +1,15 @@ +package io.opencmw.serialiser.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.TYPE }) +public @interface MetaInfo { + String unit() default ""; + String description() default ""; + String direction() default ""; + String[] groups() default ""; +} \ No newline at end of file diff --git a/serialiser/src/main/java/io/opencmw/serialiser/annotations/Unit.java b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Unit.java new file mode 100644 index 00000000..61012c1c --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/annotations/Unit.java @@ -0,0 +1,12 @@ +package io.opencmw.serialiser.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.TYPE }) +public @interface Unit { + String value() default ""; +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/BinarySerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/BinarySerialiser.java new file mode 100644 index 00000000..27fb041b --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/BinarySerialiser.java @@ -0,0 +1,1679 @@ +package io.opencmw.serialiser.spi; + +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.utils.AssertUtils; +import io.opencmw.serialiser.utils.ClassUtils; +import io.opencmw.serialiser.utils.GenericsHelper; + +/** + * YaS -- Yet another Serialiser implementation + * + * Generic binary serialiser aimed at efficiently transferring data between server/client and in particular between + * Java/C++/web-based programs. For rationale see IoSerialiser.md description. + * + *

+ * There are two default backing buffer implementations ({@link FastByteBuffer FastByteBuffer} and {@link ByteBuffer ByteBuffer}), + * but can be extended/replaced with any other buffer is also possible provided it implements the {@link IoBuffer IoBuffer} interface. + * + *

+ * The default serialisable data types are defined in {@link DataType DataType} and include definitions for + *

    + *
  • primitives (byte, short, ..., float, double, and String), and + *
  • arrays thereof (ie. byte[], short[], ..., float[], double[], and String[]), as well as + *
  • complex objects implementing Collections (ie. Set, List, Queues), Enums or Maps. + *
+ * Any other complex data objects can be stored/extended using the {@link DataType#OTHER OTHER} sub-type. + * + * N.B. Multi-dimensional arrays are handled through one-dimensional striding arrays with the additional + * infos on number of dimensions and size for each individual dimension. + * + *

+ * raw-byte level protocol: above data items are stored as follows: + * 


+ * * header info:   [ 4 bytes (int) = 0x0000002A] // magic number used as coarse protocol identifier - precise protocol refined by further fields below
+ *                  [ clear text serialiser name: String ] + // ie. "YaS" for 'Yet another Serialiser'
+ *                  [ 1 byte - major protocol version ] +
+ *                  [ 1 byte - minor protocol version ] +
+ *                  [ 1 byte - micro protocol version ] // micro: non API-changing bug fixes in implementation
+ *                  [ field header for 'start marker' ] [ 1 byte - uniqueType (0x00) ]
+ * * String:        [ 4 bytes (int) - length (including termination) ][ n bytes based on ISO-8859 or UTF-8 encoding ]
+ * * field header:  # start field header 'p0'
+ *                  [ 1 byte - uniqueType ]
+ *                  [ 4 bytes - field name hash code] // enables faster field matching
+ *                  [ 4 bytes - dataStart = n bytes until data start] // counted w.r.t. field header start
+ *                  [ 4 bytes - dataSize = n bytes for data size]
+ *                  [ String (ISO-8859) - field name ]             // optional, if there are no field name hash code collisions
+ *                  N.B. following fields are optional (detectable if buffer position smaller than 'p0' + dataStart)
+ *                  [ String (UTF-8)    - field unit ]
+ *                  [ String (UTF-8)    - field in/out direction ]
+ *                  [ String (UTF-8)    - field groups ]
+ *                  # start data = 'p0' + dataStart
+ *                  ... type specific and/or custom data serialisation
+ *                  # end data = 'p0' + dataStart + dataSize
+ * * primitives:    [ field header for 'primitive type ID'] + [ 1-8 bytes depending on DataType ]
+ * * prim. arrays:  [ array header for 'prim. type array ID'] + [   ]=1-8 bytes x N_i or more - array data depending on variable DataType ]
+ * * boxed arrays:  as above but each element cast to corresponding primitive type
+ * * array header:  [ field header (as above) ] +
+ *                      [4 bytes - number of dimensions N_d ] +
+ *                      [4 bytes x N_d - vector sizes for each dimension N_i ]  
+ * * Collection[E]:
+ * * List[]:
+ * * Queue[E]:
+ * * Set[E]:        [ array header (uniqueType= one of the Collection type IDs) ] + 
+ *                      [ 1 byte - uniqueType of E ] + [  n bytes - array of E cast to primitive type and/or string ]
+ * * Map[K,V]:      [ array header (uniqueType=0xCB) ] + [ 1 byte - uniqueType of K ] +  [ 1 byte - uniqueType of V ] +
+ *                      [ n bytes - array of K cast to primitive type and/or string ] + 
+ *                      [ n bytes - array of V cast to primitive type and/or string ]
+ * * OTHER          [ field header - uniqueByte = 0xFD ] +
+ *                      [ 1 byte - uniqueType -- custom class type definition ]
+ *                      [ String (ISO-8859) - class type name ]
+ *                      [ n bytes - custom serialisation definition ]
+ * * start marker:  [ field header for '0x00' ] // dataSize == # bytes until the corresponding end-marker start
+ * * end marker:    [ field header for '0xFE' ]
+ * 
+ * * nesting or sub-structures (ie. POJOs with sub-classes) can be achieved via:
+ * [  start marker - field name == nesting context1 ] 
+ *   [  start marker - field name == nesting context2 ]
+ *    ... 
+ *   [  end marker - field name == nesting context2 (optional name) ]
+ * [  end marker - field name == nesting context1 (optional name) ]
+ * 
+ * with
+ * T: being a generic list parameter outlined in {@link DataType DataType}
+ * K: being a generic key parameter outlined in {@link DataType DataType}
+ * V: being a generic value parameter outlined in {@link DataType DataType}
+ *
+ * + * @author rstein + */ +@SuppressWarnings({ "PMD.CommentSize", "PMD.TooManyMethods", "PMD.ExcessivePublicCount", "PMD.PrematureDeclaration", "PMD.ExcessiveClassLength", "PMD.NPathComplexity" }) // variables need to be read from stream +public class BinarySerialiser implements IoSerialiser { + public static final String UNCHECKED_CAST_SUPPRESSION = "unchecked"; + public static final int VERSION_MAGIC_NUMBER = -1; // '-1' since CmwLight cannot have a negative number of entries + public static final String PROTOCOL_NAME = "YaS"; // Yet another Serialiser implementation + public static final byte VERSION_MAJOR = 1; + public static final byte VERSION_MINOR = 0; + public static final byte VERSION_MICRO = 0; + public static final String PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL = "protocol error: serialiser lookup must not be null for DataType == OTHER"; + public static final String PROTOCOL_MISMATCH_N_ELEMENTS_HEADER = "protocol mismatch nElements header = "; + public static final String NO_SERIALISER_IMP_FOUND = "no serialiser implementation found for classType = "; + public static final String VS_ARRAY = " vs. array = "; + private static final Logger LOGGER = LoggerFactory.getLogger(BinarySerialiser.class); + private static final int ADDITIONAL_HEADER_INFO_SIZE = 1000; + private static final DataType[] BYTE_TO_DATA_TYPE = new DataType[256]; + private static final Byte[] DATA_TYPE_TO_BYTE = new Byte[256]; + public static final String VS_SHOULD_BE = "' vs. should be '"; + + static { + // static mapping of protocol bytes -- needed to be compatible with other wire protocols + BYTE_TO_DATA_TYPE[0] = DataType.START_MARKER; + + BYTE_TO_DATA_TYPE[1] = DataType.BOOL; + BYTE_TO_DATA_TYPE[2] = DataType.BYTE; + BYTE_TO_DATA_TYPE[3] = DataType.SHORT; + BYTE_TO_DATA_TYPE[4] = DataType.INT; + BYTE_TO_DATA_TYPE[5] = DataType.LONG; + BYTE_TO_DATA_TYPE[6] = DataType.FLOAT; + BYTE_TO_DATA_TYPE[7] = DataType.DOUBLE; + BYTE_TO_DATA_TYPE[8] = DataType.CHAR; + BYTE_TO_DATA_TYPE[9] = DataType.STRING; + + BYTE_TO_DATA_TYPE[101] = DataType.BOOL_ARRAY; + BYTE_TO_DATA_TYPE[102] = DataType.BYTE_ARRAY; + BYTE_TO_DATA_TYPE[103] = DataType.SHORT_ARRAY; + BYTE_TO_DATA_TYPE[104] = DataType.INT_ARRAY; + BYTE_TO_DATA_TYPE[105] = DataType.LONG_ARRAY; + BYTE_TO_DATA_TYPE[106] = DataType.FLOAT_ARRAY; + BYTE_TO_DATA_TYPE[107] = DataType.DOUBLE_ARRAY; + BYTE_TO_DATA_TYPE[108] = DataType.CHAR_ARRAY; + BYTE_TO_DATA_TYPE[109] = DataType.STRING_ARRAY; + + BYTE_TO_DATA_TYPE[200] = DataType.COLLECTION; + BYTE_TO_DATA_TYPE[201] = DataType.ENUM; + BYTE_TO_DATA_TYPE[202] = DataType.LIST; + BYTE_TO_DATA_TYPE[203] = DataType.MAP; + BYTE_TO_DATA_TYPE[204] = DataType.QUEUE; + BYTE_TO_DATA_TYPE[205] = DataType.SET; + + BYTE_TO_DATA_TYPE[0xFD] = DataType.OTHER; + BYTE_TO_DATA_TYPE[0xFE] = DataType.END_MARKER; + + for (int i = 0; i < BYTE_TO_DATA_TYPE.length; i++) { + if (BYTE_TO_DATA_TYPE[i] == null) { + continue; + } + final int id = BYTE_TO_DATA_TYPE[i].getID(); + DATA_TYPE_TO_BYTE[id] = (byte) i; + } + } + + private int bufferIncrements = ADDITIONAL_HEADER_INFO_SIZE; + private IoBuffer buffer; + private boolean putFieldMetaData = true; + private WireDataFieldDescription parent; + private WireDataFieldDescription lastFieldHeader; + private BiFunction> fieldSerialiserLookupFunction; + + /** + * @param buffer the backing IoBuffer (see e.g. {@link FastByteBuffer} or{@link ByteBuffer} + */ + public BinarySerialiser(final IoBuffer buffer) { + super(); + this.buffer = buffer; + } + + @Override + public ProtocolInfo checkHeaderInfo() { + final int magicNumber = buffer.getInt(); + if (magicNumber != VERSION_MAGIC_NUMBER) { + throw new IllegalStateException("byte buffer version magic byte incompatible: received '" + magicNumber + VS_SHOULD_BE + VERSION_MAGIC_NUMBER + "'"); + } + final String producer = buffer.getStringISO8859(); + if (!PROTOCOL_NAME.equals(producer)) { + throw new IllegalStateException("byte buffer producer name incompatible: received '" + producer + VS_SHOULD_BE + PROTOCOL_NAME + "'"); + } + final byte major = buffer.getByte(); + final byte minor = buffer.getByte(); + final byte micro = buffer.getByte(); + + final WireDataFieldDescription headerStartField = getFieldHeader(); + final ProtocolInfo header = new ProtocolInfo(this, headerStartField, producer, major, minor, micro); + + if (!header.isCompatible()) { + final String thisHeader = String.format(" serialiser: %s-v%d.%d.%d", PROTOCOL_NAME, VERSION_MAJOR, VERSION_MINOR, VERSION_MICRO); + throw new IllegalStateException("byte buffer version incompatible: received '" + header.toString() + VS_SHOULD_BE + thisHeader + "'"); + } + return header; + } + + @Override + public void setQueryFieldName(final String fieldName, final int dataStartPosition) { + if (fieldName == null || fieldName.isBlank()) { + throw new IllegalArgumentException("fieldName must not be null or blank: " + fieldName); + } + buffer.position(dataStartPosition); + } + + @Override + public int[] getArraySizeDescriptor() { + final int nDims = buffer.getInt(); // number of dimensions + final int[] ret = new int[nDims]; + for (int i = 0; i < nDims; i++) { + ret[i] = buffer.getInt(); // vector size for each dimension + } + return ret; + } + + @Override + public boolean getBoolean() { + return buffer.getBoolean(); + } + + @Override + public boolean[] getBooleanArray(final boolean[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getBooleanArray(dst, length); + } + + @Override + public IoBuffer getBuffer() { + return buffer; + } + + @Override + public void setBuffer(final IoBuffer buffer) { + this.buffer = buffer; + } + + public int getBufferIncrements() { + return bufferIncrements; + } + + public void setBufferIncrements(final int bufferIncrements) { + AssertUtils.gtEqThanZero("bufferIncrements", bufferIncrements); + this.bufferIncrements = bufferIncrements; + } + + @Override + public byte getByte() { + return buffer.getByte(); + } + + @Override + public byte[] getByteArray(final byte[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getByteArray(dst, length); + } + + @Override + public char getChar() { + return buffer.getChar(); + } + + @Override + public char[] getCharArray(final char[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getCharArray(dst, length); + } + + @Override + public Collection getCollection(final Collection collection) { + getArraySizeDescriptor(); + final int nElements = buffer.getInt(); + final DataType collectionType = getDataType(buffer.getByte()); + final DataType valueDataType = getDataType(buffer.getByte()); + + final Collection retCollection; + if (collection == null) { + switch (collectionType) { + case SET: + retCollection = new HashSet<>(nElements); + break; + case QUEUE: + retCollection = new ArrayDeque<>(nElements); + break; + case LIST: + case COLLECTION: + default: + retCollection = new ArrayList<>(nElements); + break; + } + } else { + retCollection = collection; + retCollection.clear(); + } + + if (DataType.OTHER.equals(valueDataType)) { + final String classTypeName = buffer.getStringISO8859(); + final String secondaryTypeName = buffer.getStringISO8859(); + final Type classType = ClassUtils.getClassByName(classTypeName); + final Type[] secondaryType = secondaryTypeName.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(secondaryTypeName) }; + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (serialiserLookup == null) { + throw new IllegalArgumentException(PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL); + } + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser serialiser = (FieldSerialiser) serialiserLookup.apply(classType, secondaryType); + + if (serialiser == null) { + throw new IllegalArgumentException(NO_SERIALISER_IMP_FOUND + classTypeName); + } + for (int i = 0; i < nElements; i++) { + retCollection.add(serialiser.getReturnObjectFunction().apply(this, null, null)); + } + + return retCollection; + } + + // read primitive or String value vector + final E[] values = getGenericArrayAsBoxedPrimitive(valueDataType); + if (nElements != values.length) { + throw new IllegalStateException(PROTOCOL_MISMATCH_N_ELEMENTS_HEADER + nElements + VS_ARRAY + values.length); + } + retCollection.addAll(Arrays.asList(values)); + + return retCollection; + } + + @Override + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + public E getCustomData(final FieldSerialiser serialiser) { + String classType = null; + String classSecondaryType = null; + try { + classType = buffer.getStringISO8859(); + classSecondaryType = buffer.getStringISO8859(); + if (serialiser == null) { + final Type classTypeT = ClassUtils.getClassByName(classType); + final Type[] secondaryTypeT = classSecondaryType.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(classSecondaryType) }; + return (E) getSerialiserLookupFunction().apply(classTypeT, secondaryTypeT).getReturnObjectFunction().apply(this, null, null); + } else { + return serialiser.getReturnObjectFunction().apply(this, null, null); + } + } catch (Exception e) { // NOPMD + LOGGER.atError().setCause(e).addArgument(classType).addArgument(classSecondaryType).log("problems with generic classType: {} classSecondaryType: {}"); + throw e; + } + } + + @Override + public double getDouble() { + return buffer.getDouble(); + } + + @Override + public double[] getDoubleArray(final double[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getDoubleArray(dst, length); + } + + @Override + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + public > Enum getEnum(final Enum enumeration) { + // read value vector + final String enumSimpleName = buffer.getStringISO8859(); + final String enumName = buffer.getStringISO8859(); + buffer.getStringISO8859(); // enumTypeList + final String enumState = buffer.getStringISO8859(); + buffer.getInt(); // enumOrdinal + // N.B. for the time being package name + class name is required + Class enumClass = ClassUtils.getClassByName(enumName); + if (enumClass == null) { + enumClass = ClassUtils.getClassByName(enumSimpleName); + if (enumClass == null) { + throw new IllegalStateException( + "could not find enum class description '" + enumName + "' or '" + enumSimpleName + "'"); + } + } + + try { + final Method valueOf = enumClass.getMethod("valueOf", String.class); + return (Enum) valueOf.invoke(null, enumState); + } catch (final ReflectiveOperationException e) { + LOGGER.atError().setCause(e).addArgument(enumClass).log("could not match 'valueOf(String)' function for class/(supposedly) enum of {}"); + } + + return null; + } + + @Override + public String getEnumTypeList() { + // read value vector + buffer.getStringISO8859(); // enumSimpleName + buffer.getStringISO8859(); // enumName + final String enumTypeList = buffer.getStringISO8859(); + buffer.getStringISO8859(); // enumState + buffer.getInt(); // enumOrdinal + + return enumTypeList; + } + + @Override + public WireDataFieldDescription getFieldHeader() { + final int headerStart = buffer.position(); + final byte dataTypeByte = buffer.getByte(); + final int fieldNameHashCode = buffer.getInt(); + final int dataStartOffset = buffer.getInt(); + final int dataStartPosition = headerStart + dataStartOffset; + int dataSize = buffer.getInt(); + final String fieldName; + if (buffer.position() < dataStartPosition) { + fieldName = buffer.getStringISO8859(); + } else { + fieldName = null; + } + + final DataType dataType = getDataType(dataTypeByte); + if (dataType == DataType.END_MARKER) { + parent = (WireDataFieldDescription) parent.getParent(); + } + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldNameHashCode, fieldName, dataType, headerStart, dataStartOffset, dataSize); + if (dataType == DataType.START_MARKER) { + parent = lastFieldHeader; + } + + if (this.isPutFieldMetaData()) { + // optional meta data + if (buffer.position() < dataStartPosition) { + lastFieldHeader.setFieldUnit(buffer.getString()); + } + if (buffer.position() < dataStartPosition) { + lastFieldHeader.setFieldDescription(buffer.getString()); + } + if (buffer.position() < dataStartPosition) { + lastFieldHeader.setFieldDirection(buffer.getString()); + } + if (buffer.position() < dataStartPosition) { + final String[] fieldGroups = buffer.getStringArray(); + lastFieldHeader.setFieldGroups(fieldGroups == null ? Collections.emptyList() : Arrays.asList(fieldGroups)); + } + } else { + buffer.position(dataStartPosition); + } + + // check for header-dataStart offset consistency + if (buffer.position() != dataStartPosition) { + final int diff = dataStartPosition - buffer.position(); + throw new IllegalStateException("could not parse FieldHeader: fieldName='" + dataType + ":" + fieldName + "' dataOffset = " + dataStartOffset + " bytes (read) -- " // + + " buffer position is " + buffer.position() + " vs. calculated " + dataStartPosition + " diff = " + diff); + } + + if (dataSize >= 0) { + return lastFieldHeader; + } + + // last-minute check in case dataSize hasn't been set correctly + if (dataType.isScalar()) { + dataSize = dataType.getPrimitiveSize(); + } else if (dataType == DataType.STRING) { + // sneak-peak look-ahead to get actual string size + // N.B. regarding jump size: <(>string size -1> + + dataSize = buffer.getInt(buffer.position() + FastByteBuffer.SIZE_OF_INT) + FastByteBuffer.SIZE_OF_INT; + } + lastFieldHeader.setDataSize(dataSize); + + return lastFieldHeader; + } + + @Override + public float getFloat() { + return buffer.getFloat(); + } + + @Override + public float[] getFloatArray(final float[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getFloatArray(dst, length); + } + + @Override + public int getInt() { + return buffer.getInt(); + } + + @Override + public int[] getIntArray(final int[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getIntArray(dst, length); + } + + @Override + public List getList(final List collection) { + getArraySizeDescriptor(); + final int nElements = buffer.getInt(); + final DataType listDataType = getDataType(buffer.getByte()); + final DataType valueDataType = getDataType(buffer.getByte()); + if (!listDataType.equals(DataType.LIST) && !listDataType.equals(DataType.COLLECTION)) { + throw new IllegalArgumentException("dataType incompatible with List = " + listDataType); + } + final List retCollection; + if (collection == null) { + retCollection = new ArrayList<>(); + } else { + retCollection = collection; + retCollection.clear(); + } + + if (DataType.OTHER.equals(valueDataType)) { + final String classTypeName = buffer.getStringISO8859(); + final String secondaryTypeName = buffer.getStringISO8859(); + final Type classType = ClassUtils.getClassByName(classTypeName); + final Type[] secondaryType = secondaryTypeName.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(secondaryTypeName) }; + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (serialiserLookup == null) { + throw new IllegalArgumentException(PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL); + } + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser serialiser = (FieldSerialiser) serialiserLookup.apply(classType, secondaryType); + + if (serialiser == null) { + throw new IllegalArgumentException(NO_SERIALISER_IMP_FOUND + classTypeName); + } + for (int i = 0; i < nElements; i++) { + retCollection.add(serialiser.getReturnObjectFunction().apply(this, null, null)); + } + + return retCollection; + } + + // read primitive or String value vector + final E[] values = getGenericArrayAsBoxedPrimitive(valueDataType); + if (nElements != values.length) { + throw new IllegalStateException(PROTOCOL_MISMATCH_N_ELEMENTS_HEADER + nElements + VS_ARRAY + values.length); + } + retCollection.addAll(Arrays.asList(values)); + + return retCollection; + } + + @Override + public long getLong() { + return buffer.getLong(); + } + + @Override + public long[] getLongArray(final long[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getLongArray(dst, length); + } + + @Override + @SuppressWarnings({ UNCHECKED_CAST_SUPPRESSION }) + public Map getMap(final Map map) { // NOSONAR NOPMD + getArraySizeDescriptor(); + final int nElements = buffer.getInt(); + // convert into two linear arrays one of K and the other for V streamer encoding as + // <1 (int)> + + // read key type and key value vector + final K[] keys; + final DataType keyDataType = getDataType(buffer.getByte()); + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (keyDataType == DataType.OTHER) { + final String classTypeName = buffer.getStringISO8859(); + final String secondaryTypeName = buffer.getStringISO8859(); + final Type classType = ClassUtils.getClassByName(classTypeName); + final Type[] secondaryType = secondaryTypeName.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(secondaryTypeName) }; + if (serialiserLookup == null) { + throw new IllegalArgumentException(PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL); + } + final FieldSerialiser serialiser = serialiserLookup.apply(classType, secondaryType); + if (serialiser == null) { + throw new IllegalArgumentException(NO_SERIALISER_IMP_FOUND + classTypeName); + } + keys = (K[]) new Object[nElements]; + for (int i = 0; i < keys.length; i++) { + keys[i] = (K) serialiser.getReturnObjectFunction().apply(this, null, null); + } + } else { + keys = getGenericArrayAsBoxedPrimitive(keyDataType); + } + // read value type and value vector + final V[] values; + final DataType valueDataType = getDataType(buffer.getByte()); + if (valueDataType == DataType.OTHER) { + final String classTypeName = buffer.getStringISO8859(); + final String secondaryTypeName = buffer.getStringISO8859(); + final Type classType = ClassUtils.getClassByName(classTypeName); + final Type[] secondaryType = secondaryTypeName.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(secondaryTypeName) }; + if (serialiserLookup == null) { + throw new IllegalArgumentException(PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL); + } + final FieldSerialiser serialiser = serialiserLookup.apply(classType, secondaryType); + + if (serialiser == null) { + throw new IllegalArgumentException(NO_SERIALISER_IMP_FOUND + classTypeName); + } + values = (V[]) new Object[nElements]; + for (int i = 0; i < values.length; i++) { + values[i] = (V) serialiser.getReturnObjectFunction().apply(this, null, null); + } + } else { + values = getGenericArrayAsBoxedPrimitive(valueDataType); + } + + // generate new/write into existing Map + final Map retMap = map == null ? new ConcurrentHashMap<>() : map; + if (map != null) { + map.clear(); + } + for (int i = 0; i < keys.length; i++) { + retMap.put(keys[i], values[i]); + } + + return retMap; + } + + public WireDataFieldDescription getParent() { + return parent; + } + + @Override + public Queue getQueue(final Queue collection) { + getArraySizeDescriptor(); + final int nElements = buffer.getInt(); + final DataType listDataType = getDataType(buffer.getByte()); + final DataType valueDataType = getDataType(buffer.getByte()); + if (!listDataType.equals(DataType.QUEUE) && !listDataType.equals(DataType.COLLECTION)) { + throw new IllegalArgumentException("dataType incompatible with Queue = " + listDataType); + } + final Queue retCollection; + if (collection == null) { + retCollection = new ArrayDeque<>(); + } else { + retCollection = collection; + retCollection.clear(); + } + + if (DataType.OTHER.equals(valueDataType)) { + final String classTypeName = buffer.getStringISO8859(); + final String secondaryTypeName = buffer.getStringISO8859(); + final Type classType = ClassUtils.getClassByName(classTypeName); + final Type[] secondaryType = secondaryTypeName.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(secondaryTypeName) }; + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (serialiserLookup == null) { + throw new IllegalArgumentException(PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL); + } + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser serialiser = (FieldSerialiser) serialiserLookup.apply(classType, secondaryType); + + if (serialiser == null) { + throw new IllegalArgumentException(NO_SERIALISER_IMP_FOUND + classTypeName); + } + for (int i = 0; i < nElements; i++) { + retCollection.add(serialiser.getReturnObjectFunction().apply(this, null, null)); + } + + return retCollection; + } + + // read primitive or String value vector + final E[] values = getGenericArrayAsBoxedPrimitive(valueDataType); + if (nElements != values.length) { + throw new IllegalStateException(PROTOCOL_MISMATCH_N_ELEMENTS_HEADER + nElements + VS_ARRAY + values.length); + } + retCollection.addAll(Arrays.asList(values)); + + return retCollection; + } + + @Override + public Set getSet(final Set collection) { + getArraySizeDescriptor(); + final int nElements = buffer.getInt(); + final DataType listDataType = getDataType(buffer.getByte()); + final DataType valueDataType = getDataType(buffer.getByte()); + if (!listDataType.equals(DataType.SET) && !listDataType.equals(DataType.COLLECTION)) { + throw new IllegalArgumentException("dataType incompatible with Set = " + listDataType); + } + final Set retCollection; + if (collection == null) { + retCollection = new HashSet<>(); + } else { + retCollection = collection; + retCollection.clear(); + } + + if (DataType.OTHER.equals(valueDataType)) { + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (serialiserLookup == null) { + throw new IllegalArgumentException(PROTOCOL_ERROR_SERIALISER_LOOKUP_MUST_NOT_BE_NULL); + } + final String classTypeName = buffer.getStringISO8859(); + final String secondaryTypeName = buffer.getStringISO8859(); + final Type classType = ClassUtils.getClassByName(classTypeName); + final Type[] secondaryType = secondaryTypeName.isEmpty() ? new Type[0] : new Type[] { ClassUtils.getClassByName(secondaryTypeName) }; + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser serialiser = (FieldSerialiser) serialiserLookup.apply(classType, secondaryType); + + if (serialiser == null) { + throw new IllegalArgumentException(NO_SERIALISER_IMP_FOUND + classTypeName); + } + for (int i = 0; i < nElements; i++) { + retCollection.add(serialiser.getReturnObjectFunction().apply(this, null, null)); + } + + return retCollection; + } + + // read primitive or String value vector + final E[] values = getGenericArrayAsBoxedPrimitive(valueDataType); + if (nElements != values.length) { + throw new IllegalStateException(PROTOCOL_MISMATCH_N_ELEMENTS_HEADER + nElements + VS_ARRAY + values.length); + } + retCollection.addAll(Arrays.asList(values)); + + return retCollection; + } + + @Override + public short getShort() { + return buffer.getShort(); + } + + @Override + public short[] getShortArray(final short[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getShortArray(dst, length); + } + + @Override + public String getString() { + return buffer.getString(); + } + + @Override + public String[] getStringArray(final String[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getStringArray(dst, length); + } + + @Override + public String getStringISO8859() { + return buffer.getStringISO8859(); + } + + /** + * @return {@code true} the ISO-8859-1 character encoding is being enforced for data fields (better performance), otherwise UTF-8 is being used (more generic encoding) + */ + public boolean isEnforceSimpleStringEncoding() { + return buffer.isEnforceSimpleStringEncoding(); + } + + /** + * + * @param state {@code true} the ISO-8859-1 character encoding is being enforced for data fields (better performance), otherwise UTF-8 is being used (more generic encoding) + */ + public void setEnforceSimpleStringEncoding(final boolean state) { + buffer.setEnforceSimpleStringEncoding(state); + } + + @Override + public boolean isPutFieldMetaData() { + return putFieldMetaData; + } + + @Override + public void setPutFieldMetaData(final boolean putFieldMetaData) { + this.putFieldMetaData = putFieldMetaData; + } + + @Override + public WireDataFieldDescription parseIoStream(final boolean readHeader) { + final WireDataFieldDescription fieldRoot = getRootElement(); + parent = fieldRoot; + final WireDataFieldDescription headerRoot = readHeader ? checkHeaderInfo().getFieldHeader() : getFieldHeader(); + buffer.position(headerRoot.getDataStartPosition()); + parseIoStream(headerRoot, 0); + //updateDataEndMarker(fieldRoot) + return fieldRoot; + } + + public void parseIoStream(final WireDataFieldDescription fieldRoot, final int recursionDepth) { + if (fieldRoot.getParent() == null) { + parent = lastFieldHeader = fieldRoot; + } + WireDataFieldDescription field; + while ((field = getFieldHeader()) != null) { + final DataType dataType = field.getDataType(); + if (dataType == DataType.END_MARKER) { + // reached end of (sub-)class - close nested hierarchy + break; + } + + if (dataType == DataType.START_MARKER) { + // detected sub-class start marker + parseIoStream(field, recursionDepth + 1); + continue; + } + + final int dataSize = field.getDataSize(); + if (dataSize < 0) { + throw new IllegalStateException("FieldDescription for '" + field.getFieldName() + "' type '" + dataType + "' has negative dataSize = " + dataSize); + } + final int skipPosition = field.getDataStartPosition() + dataSize; + buffer.position(skipPosition); + } + } + + @Override + public void put(final FieldDescription fieldDescription, final Collection collection, final Type valueType) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final Object[] values = collection.toArray(); + final int nElements = collection.size(); + final Class cleanedType = ClassUtils.getRawType(valueType); + final DataType valueDataType = DataType.fromClassType(cleanedType); + final int entrySize = 17; // as an initial estimate + putArraySizeDescriptor(nElements); + buffer.putInt(nElements); + + if (collection instanceof Queue) { + buffer.putByte(getDataType(DataType.QUEUE)); + } else if (collection instanceof Set) { + buffer.putByte(getDataType(DataType.SET)); + } else if (collection instanceof List) { + buffer.putByte(getDataType(DataType.LIST)); + } else { + buffer.putByte(getDataType(DataType.COLLECTION)); + } + + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (ClassUtils.isPrimitiveWrapperOrString(cleanedType) || serialiserLookup == null) { + buffer.ensureAdditionalCapacity((nElements * entrySize) + 9); + buffer.putByte(getDataType(valueDataType)); // write value element type + putGenericArrayAsPrimitive(valueDataType, values, nElements); + } else { + buffer.putByte(getDataType(DataType.OTHER)); // write value element type + final Type[] secondaryType = ClassUtils.getSecondaryType(valueType); + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final FieldSerialiser serialiser = (FieldSerialiser) serialiserLookup.apply(valueType, secondaryType); + if (serialiser == null) { + throw new IllegalArgumentException("could not find serialiser for class type " + valueType); + } + buffer.putStringISO8859(serialiser.getClassPrototype().getCanonicalName()); // primary type + buffer.putStringISO8859(serialiser.getGenericsPrototypes().isEmpty() ? "" : serialiser.getGenericsPrototypes().get(0).getTypeName()); // secondary type if any + + final FieldSerialiser.TriConsumer writerFunction = serialiser.getWriterFunction(); + for (final Object value : values) { + writerFunction.accept(this, value, null); + } + } + + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final Enum enumeration) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + if (enumeration == null) { + return; + } + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + final Class> clazz = (Class>) enumeration.getClass(); + if (clazz == null) { + return; + } + final Enum[] enumConsts = clazz.getEnumConstants(); + if (enumConsts == null) { + return; + } + + final int nElements = 1; + final int entrySize = 17; // as an initial estimate + + buffer.ensureAdditionalCapacity((nElements * entrySize) + 9); + final String typeList = Arrays.stream(clazz.getEnumConstants()).map(Object::toString).collect(Collectors.joining(", ", "[", "]")); + buffer.putStringISO8859(clazz.getSimpleName()); + buffer.putStringISO8859(enumeration.getClass().getName()); + buffer.putStringISO8859(typeList); + buffer.putStringISO8859(enumeration.name()); + buffer.putInt(enumeration.ordinal()); + + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final Map map, Type keyType, Type valueType) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final Object[] keySet = map.keySet().toArray(); + final int nElements = keySet.length; + putArraySizeDescriptor(nElements); + buffer.putInt(nElements); + + // convert into two linear arrays one of K and the other for V streamer encoding as + // <1 (int)> + + final Class cleanedKeyType = ClassUtils.getRawType(keyType); + final DataType keyDataType = DataType.fromClassType(cleanedKeyType); + final BiFunction> serialiserLookup = getSerialiserLookupFunction(); + if (serialiserLookup == null || ClassUtils.isPrimitiveWrapperOrString(cleanedKeyType)) { + final int entrySize = 17; // as an initial estimate + buffer.ensureAdditionalCapacity((nElements * entrySize) + 9); + buffer.putByte(getDataType(keyDataType)); // write key element type + putGenericArrayAsPrimitive(keyDataType, keySet, nElements); + } else { + // write key type + buffer.putByte(getDataType(DataType.OTHER)); // write key element type + final Type[] secondaryKeyType = ClassUtils.getSecondaryType(keyType); + final FieldSerialiser serialiserKey = serialiserLookup.apply(keyType, secondaryKeyType); + if (serialiserKey == null) { + throw new IllegalArgumentException("could not find serialiser for key class type " + keyType); + } + buffer.putStringISO8859(serialiserKey.getClassPrototype().getCanonicalName()); // primary type + buffer.putStringISO8859(serialiserKey.getGenericsPrototypes().isEmpty() ? "" : serialiserKey.getGenericsPrototypes().get(0).getTypeName()); // secondary key type if any + // write key data + final FieldSerialiser.TriConsumer writerFunctionKey = serialiserKey.getWriterFunction(); + for (final Object key : keySet) { + writerFunctionKey.accept(this, key, null); + } + } + + final Class cleanedValueType = ClassUtils.getRawType(valueType); + final Object[] valueSet = map.values().toArray(); + final DataType valueDataType = DataType.fromClassType(cleanedValueType); + if (serialiserLookup == null || ClassUtils.isPrimitiveWrapperOrString(cleanedValueType)) { + final int entrySize = 17; // as an initial estimate + buffer.ensureAdditionalCapacity((nElements * entrySize) + 9); + buffer.putByte(getDataType(valueDataType)); // write value element type + putGenericArrayAsPrimitive(valueDataType, valueSet, nElements); + } else { + // write value type + buffer.putByte(getDataType(DataType.OTHER)); // write key element type + final Type[] secondaryValueType = ClassUtils.getSecondaryType(valueType); + final FieldSerialiser serialiserValue = serialiserLookup.apply(valueType, secondaryValueType); + if (serialiserValue == null) { + throw new IllegalArgumentException("could not find serialiser for value class type " + valueType); + } + buffer.putStringISO8859(serialiserValue.getClassPrototype().getCanonicalName()); // primary type + buffer.putStringISO8859(serialiserValue.getGenericsPrototypes().isEmpty() ? "" : serialiserValue.getGenericsPrototypes().get(0).getTypeName()); // secondary key type if any + + // write key data + final FieldSerialiser.TriConsumer writerFunctionValue = serialiserValue.getWriterFunction(); + for (final Object value : valueSet) { + writerFunctionValue.accept(this, value, null); + } + } + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final Collection collection, final Type valueType) { + final DataType dataType; + if (collection instanceof Queue) { + dataType = DataType.QUEUE; + } else if (collection instanceof Set) { + dataType = DataType.SET; + } else if (collection instanceof List) { + dataType = DataType.LIST; + } else { + dataType = DataType.COLLECTION; + } + + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, dataType); + this.put((FieldDescription) null, collection, valueType); + this.updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final Enum enumeration) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.ENUM); + this.put((FieldDescription) null, enumeration); + this.updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final Map map, final Type keyType, final Type valueType) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.MAP); + this.put((FieldDescription) null, map, keyType, valueType); + this.updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean value) { + this.putFieldHeader(fieldDescription); + buffer.putBoolean(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte value) { + this.putFieldHeader(fieldDescription); + buffer.putByte(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final char value) { + this.putFieldHeader(fieldDescription); + buffer.putChar(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final char[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final char[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final double value) { + this.putFieldHeader(fieldDescription); + buffer.putDouble(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final double[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final double[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final float value) { + this.putFieldHeader(fieldDescription); + buffer.putFloat(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final float[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final float[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final int value) { + this.putFieldHeader(fieldDescription); + buffer.putInt(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final int[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final int[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final long value) { + this.putFieldHeader(fieldDescription); + buffer.putLong(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final long[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final long[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final short value) { // NOPMD by rstein + this.putFieldHeader(fieldDescription); + buffer.putShort(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final short[] values, final int n) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final short[] values, final int[] dims) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final String string) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + buffer.putString(string); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final String[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int valuesSize = values == null ? 0 : values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final String[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription); + final int nElements = putArraySizeDescriptor(dims); + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final boolean value) { + this.putFieldHeader(fieldName, DataType.BOOL); + buffer.putBoolean(value); + } + + @Override + public void put(final String fieldName, final boolean[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BOOL_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final boolean[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BOOL_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final byte value) { + this.putFieldHeader(fieldName, DataType.BYTE); + buffer.putByte(value); + } + + @Override + public void put(final String fieldName, final byte[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BYTE_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final byte[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BYTE_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final char value) { + this.putFieldHeader(fieldName, DataType.CHAR); + buffer.putChar(value); + } + + @Override + public void put(final String fieldName, final char[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.CHAR_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final char[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.CHAR_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final double value) { + this.putFieldHeader(fieldName, DataType.DOUBLE); + buffer.putDouble(value); + } + + @Override + public void put(final String fieldName, final double[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.DOUBLE_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final double[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.DOUBLE_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final float value) { + this.putFieldHeader(fieldName, DataType.FLOAT); + buffer.putFloat(value); + } + + @Override + public void put(final String fieldName, final float[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.FLOAT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final float[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.FLOAT_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final int value) { + this.putFieldHeader(fieldName, DataType.INT); + buffer.putInt(value); + } + + @Override + public void put(final String fieldName, final int[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.INT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final int[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.INT_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final long value) { + this.putFieldHeader(fieldName, DataType.LONG); + buffer.putLong(value); + } + + @Override + public void put(final String fieldName, final long[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.LONG_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final long[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.LONG_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final short value) { // NOPMD by rstein + this.putFieldHeader(fieldName, DataType.SHORT); + buffer.putShort(value); + } + + @Override + public void put(final String fieldName, final short[] values, final int n) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.SHORT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final short[] values, final int[] dims) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.SHORT_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final String string) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.STRING); + buffer.putString(string); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final String[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.STRING_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final String[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.STRING_ARRAY); + final int nElements = putArraySizeDescriptor(dims); + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public int putArraySizeDescriptor(final int n) { + buffer.putInt(1); // number of dimensions + buffer.putInt(n); // vector size for each dimension + return n; + } + + @Override + public int putArraySizeDescriptor(final int[] dims) { + buffer.putInt(dims.length); // number of dimensions + int nElements = 1; + for (final int dim : dims) { + nElements *= dim; + buffer.putInt(dim); // vector size for each dimension + } + return nElements; + } + + @Override + public WireDataFieldDescription putCustomData(final FieldDescription fieldDescription, final E rootObject, Class type, final FieldSerialiser serialiser) { + if (parent == null) { + parent = lastFieldHeader = getRootElement(); + } + final WireDataFieldDescription oldParent = parent; + final WireDataFieldDescription ret = putFieldHeader(fieldDescription); + buffer.putByte(ret.getFieldStart(), getDataType(DataType.OTHER)); + parent = lastFieldHeader; + // write generic class description and type arguments (if any) to aid reconstruction + buffer.putStringISO8859(serialiser.getClassPrototype().getCanonicalName()); // primary type + buffer.putStringISO8859(serialiser.getGenericsPrototypes().isEmpty() ? "" : serialiser.getGenericsPrototypes().get(0).getTypeName()); // secondary type if any + serialiser.getWriterFunction().accept(this, rootObject, fieldDescription instanceof ClassFieldDescription ? (ClassFieldDescription) fieldDescription : null); + putEndMarker(fieldDescription); + parent = oldParent; + return ret; + } + + @Override + public void putEndMarker(final FieldDescription fieldDescription) { + updateDataEndMarker(parent); + updateDataEndMarker(lastFieldHeader); + if (parent.getParent() != null) { + parent = (WireDataFieldDescription) parent.getParent(); + } + + putFieldHeader(fieldDescription); + buffer.putByte(lastFieldHeader.getFieldStart(), getDataType(DataType.END_MARKER)); + } + + @Override + public WireDataFieldDescription putFieldHeader(final FieldDescription fieldDescription) { + if (fieldDescription == null) { + // early return + return null; + } + final DataType dataType = fieldDescription.getDataType(); + if (isPutFieldMetaData()) { + buffer.ensureAdditionalCapacity(bufferIncrements); + } + final boolean isScalar = dataType.isScalar(); + + // -- offset 0 vs. field start + final int headerStart = buffer.position(); + buffer.putByte(getDataType(dataType)); // data type ID + buffer.putInt(fieldDescription.getFieldNameHashCode()); + buffer.putInt(-1); // dataStart offset + final int dataSize = isScalar ? dataType.getPrimitiveSize() : -1; + buffer.putInt(dataSize); // dataSize (N.B. 'headerStart' + 'dataStart + dataSize' == start of next field header + buffer.putStringISO8859(fieldDescription.getFieldName()); // full field name + + if (isPutFieldMetaData() && fieldDescription.isAnnotationPresent() && dataType != DataType.END_MARKER) { + buffer.putString(fieldDescription.getFieldUnit()); + buffer.putString(fieldDescription.getFieldDescription()); + buffer.putString(fieldDescription.getFieldDirection()); + final String[] groups = fieldDescription.getFieldGroups().toArray(new String[0]); + buffer.putStringArray(groups, groups.length); + } + + // -- offset dataStart calculations + final int dataStartOffset = buffer.position() - headerStart; + buffer.putInt(headerStart + 5, dataStartOffset); // write offset to dataStart + + // from hereon there are data specific structures + buffer.ensureAdditionalCapacity(16); // allocate 16 bytes to account for potential array header (safe-bet) + + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldDescription.getFieldNameHashCode(), fieldDescription.getFieldName(), dataType, headerStart, dataStartOffset, dataSize); + if (isPutFieldMetaData() && fieldDescription.isAnnotationPresent()) { + lastFieldHeader.setFieldUnit(fieldDescription.getFieldUnit()); + lastFieldHeader.setFieldDescription(fieldDescription.getFieldDescription()); + lastFieldHeader.setFieldDirection(fieldDescription.getFieldDirection()); + lastFieldHeader.setFieldGroups(fieldDescription.getFieldGroups()); + } + return lastFieldHeader; + } + + @Override + public WireDataFieldDescription putFieldHeader(final String fieldName, final DataType dataType) { + final int addCapacity = ((fieldName.length() + 18) * FastByteBuffer.SIZE_OF_BYTE) + bufferIncrements + dataType.getPrimitiveSize(); + buffer.ensureAdditionalCapacity(addCapacity); + final boolean isScalar = dataType.isScalar(); + + // -- offset 0 vs. field start + final int headerStart = buffer.position(); + buffer.putByte(getDataType(dataType)); // data type ID + buffer.putInt(fieldName.hashCode()); // unique hashCode identifier -- TODO: unify across C++/Java & optimise performance + buffer.putInt(-1); // dataStart offset + final int dataSize = isScalar ? dataType.getPrimitiveSize() : -1; + buffer.putInt(dataSize); // dataSize (N.B. 'headerStart' + 'dataStart + dataSize' == start of next field header + buffer.putStringISO8859(fieldName); // full field name + + // this putField method cannot add meta-data use 'putFieldHeader(final FieldDescription fieldDescription)' instead + + // -- offset dataStart calculations + final int fieldHeaderDataStart = buffer.position(); + final int dataStartOffset = (fieldHeaderDataStart - headerStart); + buffer.putInt(headerStart + 5, dataStartOffset); // write offset to dataStart + + // from hereon there are data specific structures + buffer.ensureAdditionalCapacity(16); // allocate 16 bytes to account for potential array header (safe-bet) + + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldName.hashCode(), fieldName, dataType, headerStart, dataStartOffset, dataSize); + return lastFieldHeader; + } + + public void putGenericArrayAsPrimitive(final DataType dataType, final Object[] data, final int nToCopy) { + putArraySizeDescriptor(nToCopy); + switch (dataType) { + case BOOL: + buffer.putBooleanArray(GenericsHelper.toBoolPrimitive(data), nToCopy); + break; + case BYTE: + buffer.putByteArray(GenericsHelper.toBytePrimitive(data), nToCopy); + break; + case CHAR: + buffer.putCharArray(GenericsHelper.toCharPrimitive(data), nToCopy); + break; + case SHORT: + buffer.putShortArray(GenericsHelper.toShortPrimitive(data), nToCopy); + break; + case INT: + buffer.putIntArray(GenericsHelper.toIntegerPrimitive(data), nToCopy); + break; + case LONG: + buffer.putLongArray(GenericsHelper.toLongPrimitive(data), nToCopy); + break; + case FLOAT: + buffer.putFloatArray(GenericsHelper.toFloatPrimitive(data), nToCopy); + break; + case DOUBLE: + buffer.putDoubleArray(GenericsHelper.toDoublePrimitive(data), nToCopy); + break; + case STRING: + buffer.putStringArray(GenericsHelper.toStringPrimitive(data), nToCopy); + break; + case OTHER: + break; + default: + throw new IllegalArgumentException("type not implemented - " + data[0].getClass().getSimpleName()); + } + } + + @Override + public void putHeaderInfo(final FieldDescription... field) { + parent = lastFieldHeader = getRootElement(); + + buffer.ensureAdditionalCapacity(ADDITIONAL_HEADER_INFO_SIZE); + buffer.putInt(VERSION_MAGIC_NUMBER); + buffer.putStringISO8859(PROTOCOL_NAME); + buffer.putByte(VERSION_MAJOR); + buffer.putByte(VERSION_MINOR); + buffer.putByte(VERSION_MICRO); + if (field.length == 0 || field[0] == null) { + putStartMarker(new WireDataFieldDescription(this, null, "OBJ_ROOT_START".hashCode(), "OBJ_ROOT_START", DataType.START_MARKER, -1, -1, -1)); + } else { + putStartMarker(field[0]); + } + } + + @Override + public void putStartMarker(final FieldDescription fieldDescription) { + putFieldHeader(fieldDescription); + buffer.putByte(lastFieldHeader.getFieldStart(), getDataType(DataType.START_MARKER)); + + parent = lastFieldHeader; + } + + @Override + public void updateDataEndMarker(final WireDataFieldDescription fieldHeader) { + if (fieldHeader == null) { + // N.B. early return in case field header hasn't been written + return; + } + final int sizeMarkerEnd = buffer.position(); + if (isPutFieldMetaData() && sizeMarkerEnd >= buffer.capacity()) { + throw new IllegalStateException("buffer position " + sizeMarkerEnd + " is beyond buffer capacity " + buffer.capacity()); + } + + final int dataSize = sizeMarkerEnd - fieldHeader.getDataStartPosition(); + if (fieldHeader.getDataSize() != dataSize) { + final int headerStart = fieldHeader.getFieldStart(); + fieldHeader.setDataSize(dataSize); + buffer.putInt(headerStart + 9, dataSize); // 9 bytes = 1 byte for dataType, 4 bytes for fieldNameHashCode, 4 bytes for dataOffset + } + } + + @Override + public void setFieldSerialiserLookupFunction(final BiFunction> serialiserLookupFunction) { + this.fieldSerialiserLookupFunction = serialiserLookupFunction; + } + + @Override + public BiFunction> getSerialiserLookupFunction() { + return fieldSerialiserLookupFunction; + } + + @SuppressWarnings(UNCHECKED_CAST_SUPPRESSION) + protected E[] getGenericArrayAsBoxedPrimitive(final DataType dataType) { + final Object[] retVal; + getArraySizeDescriptor(); + switch (dataType) { + case BOOL: + retVal = GenericsHelper.toObject(buffer.getBooleanArray()); + break; + case BYTE: + retVal = GenericsHelper.toObject(buffer.getByteArray()); + break; + case CHAR: + retVal = GenericsHelper.toObject(buffer.getCharArray()); + break; + case SHORT: + retVal = GenericsHelper.toObject(buffer.getShortArray()); + break; + case INT: + retVal = GenericsHelper.toObject(buffer.getIntArray()); + break; + case LONG: + retVal = GenericsHelper.toObject(buffer.getLongArray()); + break; + case FLOAT: + retVal = GenericsHelper.toObject(buffer.getFloatArray()); + break; + case DOUBLE: + retVal = GenericsHelper.toObject(buffer.getDoubleArray()); + break; + case STRING: + retVal = buffer.getStringArray(); + break; + default: + throw new IllegalArgumentException("type not implemented - " + dataType); + } + return (E[]) retVal; + } + + private WireDataFieldDescription getRootElement() { + final int headerOffset = 1 + PROTOCOL_NAME.length() + 3; // unique byte + protocol length + 3 x byte for version + return new WireDataFieldDescription(this, null, "ROOT".hashCode(), "ROOT", DataType.OTHER, buffer.position() + headerOffset, -1, -1); + } + + public static byte getDataType(final DataType dataType) { + final int id = dataType.getID(); + if (DATA_TYPE_TO_BYTE[id] != null) { + return DATA_TYPE_TO_BYTE[id]; + } + + throw new IllegalArgumentException("DataType " + dataType + " not mapped to specific byte"); + } + + public static DataType getDataType(final byte byteValue) { + final int id = byteValue & 0xFF; + if (DATA_TYPE_TO_BYTE[id] != null) { + return BYTE_TO_DATA_TYPE[id]; + } + + throw new IllegalArgumentException("DataType byteValue=" + byteValue + " rawByteValue=" + (byteValue & 0xFF) + " not mapped"); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/ByteBuffer.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/ByteBuffer.java new file mode 100644 index 00000000..a9079be4 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/ByteBuffer.java @@ -0,0 +1,599 @@ +package io.opencmw.serialiser.spi; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import io.opencmw.serialiser.IoBuffer; + +/** + * @author rstein + */ +@SuppressWarnings({ "PMD.TooManyMethods", "PMD.ExcessivePublicCount" }) // unavoidable: each primitive type needs to handled individually (no templates) +public class ByteBuffer implements IoBuffer { + public static final int SIZE_OF_BOOLEAN = 1; + public static final int SIZE_OF_BYTE = 1; + public static final int SIZE_OF_SHORT = 2; + public static final int SIZE_OF_CHAR = 2; + public static final int SIZE_OF_INT = 4; + public static final int SIZE_OF_LONG = 8; + public static final int SIZE_OF_FLOAT = 4; + public static final int SIZE_OF_DOUBLE = 8; + private static final int DEFAULT_INITIAL_CAPACITY = 1000; + private final ReadWriteLock internalLock = new ReentrantReadWriteLock(); + private final java.nio.ByteBuffer nioByteBuffer; + private boolean enforceSimpleStringEncoding; + + /** + * construct new java.nio.ByteBuffer-based ByteBuffer with DEFAULT_INITIAL_CAPACITY + */ + public ByteBuffer() { + this(DEFAULT_INITIAL_CAPACITY); + } + + /** + * construct new java.nio.ByteBuffer-based ByteBuffer with DEFAULT_INITIAL_CAPACITY + * + * @param nCapacity initial capacity + */ + public ByteBuffer(final int nCapacity) { + nioByteBuffer = java.nio.ByteBuffer.wrap(new byte[nCapacity]); + nioByteBuffer.mark(); + } + + @Override + public int capacity() { + return nioByteBuffer.capacity(); + } + + @Override + public void clear() { + nioByteBuffer.clear(); + } + + @Override + public byte[] elements() { + return nioByteBuffer.array(); + } + + @Override + public void ensureAdditionalCapacity(final int capacity) { + /* not implemented */ + } + + @Override + public void ensureCapacity(final int capacity) { + /* not implemented */ + } + + @Override + public void flip() { + nioByteBuffer.flip(); + } + + @Override + public void forceCapacity(final int length, final int preserve) { + /* not implemented */ + } + + @Override + public boolean getBoolean() { + return nioByteBuffer.get() > 0; + } + + @Override + public boolean getBoolean(final int position) { + return nioByteBuffer.get(position) > 0; + } + + @Override + public boolean[] getBooleanArray(final boolean[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final boolean[] ret = initNeeded ? new boolean[arraySize] : dst; + + for (int i = 0; i < arraySize; i++) { + ret[i] = getBoolean(); + } + return ret; + } + + @Override + public byte getByte() { + return nioByteBuffer.get(); + } + + @Override + public byte getByte(final int position) { + return nioByteBuffer.get(position); + } + + @Override + public byte[] getByteArray(final byte[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final byte[] ret = initNeeded ? new byte[arraySize] : dst; + nioByteBuffer.get(ret, 0, arraySize); + return ret; + } + + @Override + public char getChar() { + return nioByteBuffer.getChar(); + } + + @Override + public char getChar(final int position) { + return nioByteBuffer.getChar(position); + } + + @Override + public char[] getCharArray(final char[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final char[] ret = initNeeded ? new char[arraySize] : dst; + for (int i = 0; i < arraySize; i++) { + ret[i] = getChar(); + } + return ret; + } + + @Override + public double getDouble() { + return nioByteBuffer.getDouble(); + } + + @Override + public double getDouble(final int position) { + return nioByteBuffer.getDouble(position); + } + + @Override + public double[] getDoubleArray(final double[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final double[] ret = initNeeded ? new double[arraySize] : dst; + for (int i = 0; i < arraySize; i++) { + ret[i] = getDouble(); + } + return ret; + } + + @Override + public float getFloat() { + return nioByteBuffer.getFloat(); + } + + @Override + public float getFloat(final int position) { + return nioByteBuffer.getFloat(position); + } + + @Override + public float[] getFloatArray(final float[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final float[] ret = initNeeded ? new float[arraySize] : dst; + for (int i = 0; i < arraySize; i++) { + ret[i] = getFloat(); + } + return ret; + } + + @Override + public int getInt() { + return nioByteBuffer.getInt(); + } + + @Override + public int getInt(final int position) { + return nioByteBuffer.getInt(position); + } + + @Override + public int[] getIntArray(final int[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final int[] ret = initNeeded ? new int[arraySize] : dst; + for (int i = 0; i < arraySize; i++) { + ret[i] = getInt(); + } + return ret; + } + + @Override + public long getLong() { + return nioByteBuffer.getLong(); + } + + @Override + public long getLong(final int position) { + return nioByteBuffer.getLong(position); + } + + @Override + public long[] getLongArray(final long[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final long[] ret = initNeeded ? new long[arraySize] : dst; + for (int i = 0; i < arraySize; i++) { + ret[i] = getLong(); + } + return ret; + } + + @Override + public short getShort() { // NOPMD + return nioByteBuffer.getShort(); + } + + @Override + public short getShort(final int position) { + return nioByteBuffer.getShort(position); + } + + @Override + public short[] getShortArray(final short[] dst, final int length) { // NOPMD + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final short[] ret = initNeeded ? new short[arraySize] : dst; // NOPMD + for (int i = 0; i < arraySize; i++) { + ret[i] = getShort(); + } + return ret; + } + + @Override + public String getString() { + final int arraySize = getInt() - 1; // for C++ zero terminated string + final byte[] values = new byte[arraySize]; + nioByteBuffer.get(values, 0, arraySize); + getByte(); // For C++ zero terminated string + return new String(values, 0, arraySize, StandardCharsets.UTF_8); + } + + @Override + public String getString(final int position) { + final int oldPosition = nioByteBuffer.position(); + nioByteBuffer.position(position); + final String ret = getString(); + nioByteBuffer.position(oldPosition); + return ret; + } + + @Override + public String[] getStringArray(final String[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final String[] ret = initNeeded ? new String[arraySize] : dst; + for (int k = 0; k < arraySize; k++) { + ret[k] = getString(); + } + return ret; + } + + @Override + public String getStringISO8859() { + final int arraySize = getInt() - 1; // for C++ zero terminated string + final byte[] values = new byte[arraySize]; + nioByteBuffer.get(values, 0, arraySize); + getByte(); // For C++ zero terminated string + return new String(values, 0, arraySize, StandardCharsets.ISO_8859_1); + } + + @Override + public boolean hasRemaining() { + return nioByteBuffer.hasRemaining(); + } + + @Override + public boolean isEnforceSimpleStringEncoding() { + return enforceSimpleStringEncoding; + } + + @Override + public void setEnforceSimpleStringEncoding(final boolean state) { + this.enforceSimpleStringEncoding = state; + } + + @Override + public boolean isReadOnly() { + return nioByteBuffer.isReadOnly(); + } + + @Override + public int limit() { + return nioByteBuffer.limit(); + } + + @Override + public void limit(final int newLimit) { + nioByteBuffer.limit(newLimit); + } + + @Override + public ReadWriteLock lock() { + return internalLock; + } + + @Override + public int position() { + return nioByteBuffer.position(); + } + + @Override + public void position(final int newPosition) { + nioByteBuffer.position(newPosition); + } + + @Override + public void putBoolean(final boolean value) { + putByte((byte) (value ? 1 : 0)); + } + + @Override + public void putBoolean(final int position, final boolean value) { + nioByteBuffer.put(position, (byte) (value ? 1 : 0)); + } + + @Override + public void putBooleanArray(final boolean[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + putBoolean(src[i]); + } + } + + @Override + public void putByte(final byte b) { + nioByteBuffer.put(b); + } + + @Override + public void putByte(final int position, final byte value) { + nioByteBuffer.put(position, value); + } + + @Override + public void putByteArray(final byte[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = (n >= 0 ? Math.min(n, srcSize) : srcSize); + ensureAdditionalCapacity(nElements); + putInt(nElements); // strided-array size + if (src == null) { + return; + } + nioByteBuffer.put(src, 0, nElements); + } + + @Override + public void putChar(final char value) { + nioByteBuffer.putChar(value); + } + + @Override + public void putChar(final int position, final char value) { + nioByteBuffer.putChar(position, value); + } + + @Override + public void putCharArray(final char[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements * SIZE_OF_CHAR); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + nioByteBuffer.putChar(src[i]); + } + } + + @Override + public void putDouble(final double value) { + nioByteBuffer.putDouble(value); + } + + @Override + public void putDouble(final int position, final double value) { + nioByteBuffer.putDouble(position, value); + } + + @Override + public void putDoubleArray(final double[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements * SIZE_OF_DOUBLE); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + nioByteBuffer.putDouble(src[i]); + } + } + + @Override + public void putFloat(final float value) { + nioByteBuffer.putFloat(value); + } + + @Override + public void putFloat(final int position, final float value) { + nioByteBuffer.putFloat(position, value); + } + + @Override + public void putFloatArray(final float[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements * SIZE_OF_FLOAT); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + nioByteBuffer.putFloat(src[i]); + } + } + + @Override + public void putInt(final int value) { + nioByteBuffer.putInt(value); + } + + @Override + public void putInt(final int position, final int value) { + nioByteBuffer.putInt(position, value); + } + + @Override + public void putIntArray(final int[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements * SIZE_OF_INT); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + nioByteBuffer.putInt(src[i]); + } + } + + @Override + public void putLong(final long value) { + nioByteBuffer.putLong(value); + } + + @Override + public void putLong(final int position, final long value) { + nioByteBuffer.putLong(position, value); + } + + @Override + public void putLongArray(final long[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements * SIZE_OF_LONG); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + nioByteBuffer.putLong(src[i]); + } + } + + @Override + public void putShort(final short value) { // NOPMD + nioByteBuffer.putShort(value); + } + + @Override + public void putShort(final int position, final short value) { + nioByteBuffer.putShort(position, value); + } + + @Override + public void putShortArray(final short[] src, final int n) { // NOPMD + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + ensureAdditionalCapacity(nElements * SIZE_OF_SHORT); + putInt(nElements); + if (src == null) { + return; + } + for (int i = 0; i < nElements; i++) { + nioByteBuffer.putShort(src[i]); + } + } + + @Override + public void putString(final String string) { + if (isEnforceSimpleStringEncoding()) { + this.putStringISO8859(string); + return; + } + + if (string == null) { + putInt(1); // for C++ zero terminated string$ + putByte((byte) 0); // For C++ zero terminated string + return; + } + + final byte[] bytes = string.getBytes(StandardCharsets.UTF_8); + putInt(bytes.length + 1); // for C++ zero terminated string$ + ensureAdditionalCapacity(bytes.length + 1); + nioByteBuffer.put(bytes, 0, bytes.length); + putByte((byte) 0); // For C++ zero terminated string + } + + @Override + public void putString(final int position, final String value) { + final int oldPosition = nioByteBuffer.position(); + nioByteBuffer.position(position); + putString(value); + nioByteBuffer.position(oldPosition); + } + + @Override + public void putStringArray(final String[] src, final int n) { + final int srcSize = src == null ? 0 : src.length; + final int nElements = n >= 0 ? Math.min(n, srcSize) : srcSize; + putInt(nElements); + if (src == null) { + return; + } + if (isEnforceSimpleStringEncoding()) { + for (int k = 0; k < nElements; k++) { + putStringISO8859(src[k]); + } + return; + } + for (int k = 0; k < nElements; k++) { + putString(src[k]); + } + } + + @Override + public void putStringISO8859(final String string) { + final int strLength = string == null ? 0 : string.length(); + putInt(strLength + 1); // for C++ zero terminated string$ + for (int i = 0; i < strLength; ++i) { + putByte((byte) (string.charAt(i) & 0xFF)); // ISO-8859-1 encoding + } + putByte((byte) 0); // For C++ zero terminated string + } + + @Override + public int remaining() { + return nioByteBuffer.remaining(); + } + + @Override + public void reset() { + nioByteBuffer.position(0); + nioByteBuffer.mark(); + nioByteBuffer.limit(nioByteBuffer.capacity()); + nioByteBuffer.reset(); + nioByteBuffer.mark(); + } + + @Override + public void trim() { + /* not implemented */ + } + + @Override + public void trim(final int requestedCapacity) { + /* not implemented */ + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/ClassFieldDescription.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/ClassFieldDescription.java new file mode 100644 index 00000000..d74b7887 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/ClassFieldDescription.java @@ -0,0 +1,862 @@ +package io.opencmw.serialiser.spi; + +import java.lang.reflect.AnnotatedElement; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.nio.CharBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.annotations.Description; +import io.opencmw.serialiser.annotations.Direction; +import io.opencmw.serialiser.annotations.Groups; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.serialiser.annotations.Unit; +import io.opencmw.serialiser.utils.ClassUtils; + +import sun.misc.Unsafe; // NOPMD - there is nothing more suitable under the Sun + +/** + * @author rstein + */ +@SuppressWarnings({ "PMD.ExcessivePublicCount", "PMD.TooManyFields" }) // utility class for safe reflection handling +public class ClassFieldDescription implements FieldDescription { + private static final Logger LOGGER = LoggerFactory.getLogger(ClassFieldDescription.class); + private final int hierarchyDepth; + private final FieldAccess fieldAccess; + private final String fieldName; + private final int fieldNameHashCode; + private final String fieldNameRelative; + private final String fieldUnit; + private final String fieldDescription; + private final String fieldDirection; + private final List fieldGroups; + private final boolean annotationPresent; + private final ClassFieldDescription parent; + private final List children = new ArrayList<>(); + private final Class classType; + private final DataType dataType; + private final String typeName; + private final String typeNameSimple; + private final int modifierID; + private final boolean modPublic; + private final boolean modProtected; + private final boolean modPrivate; + // field modifier in canonical order + private final boolean modAbstract; + private final boolean modStatic; + private final boolean modFinal; + private final boolean modTransient; + private final boolean modVolatile; + private final boolean modSynchronized; + private final boolean modNative; + private final boolean modStrict; + private final boolean modInterface; + // additional qualities + private final boolean isPrimitiveType; + private final boolean isClassType; + private final boolean isEnumType; + private final List enumDefinitions; + private final boolean serializable; + private String toStringName; // computed on demand and cached + private Type genericType; // computed on demand and cached + private List genericTypeList; // computed on demand and cached + private List genericTypeNameList; // computed on demand and cached + private String genericTypeNames; // computed on demand and cached + private String genericTypeNamesSimple; // computed on demand and cached + private String modifierStr; // computed on demand and cached + // serialiser info + private FieldSerialiser fieldSerialiser; + + /** + * This should be called only once with the root class as an argument + * + * @param referenceClass the root node containing further Field children + * @param fullScan {@code true} if the class field should be serialised according to {@link java.io.Serializable} + * (ie. object's non-static and non-transient fields); {@code false} otherwise. + */ + public ClassFieldDescription(final Class referenceClass, final boolean fullScan) { + this(referenceClass, null, null, 0); + if (referenceClass == null) { + throw new IllegalArgumentException("object must not be null"); + } + + genericType = classType.getGenericSuperclass(); + + // parse object + exploreClass(classType, this, 0, fullScan); + } + + protected ClassFieldDescription(final Class referenceClass, final Field field, final ClassFieldDescription parent, final int recursionLevel) { + super(); + hierarchyDepth = recursionLevel; + this.parent = parent; + + if (referenceClass == null) { + if (field == null) { + throw new IllegalArgumentException("field must not be null"); + } + fieldAccess = new FieldAccess(field); + classType = field.getType(); + fieldNameHashCode = field.getName().hashCode(); + fieldName = field.getName().intern(); + fieldNameRelative = this.parent == null ? fieldName : (this.parent.getFieldNameRelative() + "." + fieldName).intern(); + + modifierID = field.getModifiers(); + dataType = DataType.fromClassType(classType); + } else { + fieldAccess = null; // it's a root, no field definition available + classType = referenceClass; + fieldNameHashCode = classType.getName().hashCode(); + fieldName = classType.getName().intern(); + fieldNameRelative = fieldName; + + modifierID = classType.getModifiers(); + dataType = DataType.START_MARKER; + } + + // read annotation values + AnnotatedElement annotatedElement = field == null ? referenceClass : field; + fieldUnit = getFieldUnit(annotatedElement); + fieldDescription = getFieldDescription(annotatedElement); + fieldDirection = getFieldDirection(annotatedElement); + fieldGroups = getFieldGroups(annotatedElement); + + annotationPresent = fieldUnit != null || fieldDescription != null || fieldDirection != null || !fieldGroups.isEmpty(); + + typeName = ClassUtils.translateClassName(classType.getTypeName()).intern(); + final int lastDot = typeName.lastIndexOf('.'); + typeNameSimple = typeName.substring(lastDot < 0 ? 0 : lastDot + 1); + + modPublic = Modifier.isPublic(modifierID); + modProtected = Modifier.isProtected(modifierID); + modPrivate = Modifier.isPrivate(modifierID); + + modAbstract = Modifier.isAbstract(modifierID); + modStatic = Modifier.isStatic(modifierID); + modFinal = Modifier.isFinal(modifierID); + modTransient = Modifier.isTransient(modifierID); + modVolatile = Modifier.isVolatile(modifierID); + modSynchronized = Modifier.isSynchronized(modifierID); + modNative = Modifier.isNative(modifierID); + modStrict = Modifier.isStrict(modifierID); + modInterface = classType.isInterface(); + + // additional fields + isPrimitiveType = classType.isPrimitive(); + isClassType = !isPrimitiveType && !modInterface; + isEnumType = Enum.class.isAssignableFrom(classType); + if (isEnumType) { + enumDefinitions = List.of(classType.getEnumConstants()); + } else { + enumDefinitions = Collections.emptyList(); + } + serializable = !modTransient && !modStatic; + } + + /** + * This should be called for individual class field members + * + * @param field Field reference for the given class member + * @param parent pointer to the root/parent reference class field description + * @param recursionLevel hierarchy level (i.e. '0' being the root class, '1' the sub-class etc. + * @param fullScan {@code true} if the class field should be serialised according to {@link java.io.Serializable} + * (ie. object's non-static and non-transient fields); {@code false} otherwise. + */ + public ClassFieldDescription(final Field field, final ClassFieldDescription parent, final int recursionLevel, + final boolean fullScan) { + this(null, field, parent, recursionLevel); + if (field == null) { + throw new IllegalArgumentException("field must not be null"); + } + + if (serializable) { + // enable access by default (saves performance later on) + field.setAccessible(true); // NOSONAR NOPMD + } + + // add child to parent if it serializable or if a full scan is requested + if (this.parent != null && (serializable || fullScan)) { + this.parent.getChildren().add(this); + } + } + + public Object allocateMemberClassField(final Object fieldParent) { + try { + // need to allocate new class object + Class fieldParentClass = ClassUtils.getRawType(getParent(this, 1).getType()); + final Object newFieldObj; + if (fieldParentClass.getDeclaringClass() == null) { + final Constructor constr = fieldParentClass.getDeclaredConstructor(); + newFieldObj = constr.newInstance(); + } else { + final Constructor constr = fieldParentClass.getDeclaredConstructor(fieldParent.getClass()); + newFieldObj = constr.newInstance(fieldParent); + } + this.getField().set(fieldParent, newFieldObj); + + return newFieldObj; + } catch (InstantiationException | InvocationTargetException | SecurityException | NoSuchMethodException | IllegalAccessException e) { + LOGGER.atError().setCause(e).log("error initialising inner class object"); + } + return null; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FieldDescription)) { + return false; + } + final FieldDescription other = (FieldDescription) obj; + if (this.getFieldNameHashCode() != other.getFieldNameHashCode()) { + return false; + } + + if (this.getDataType() != other.getDataType()) { + return false; + } + + return this.getFieldName().equals(other.getFieldName()); + } + + @Override + public FieldDescription findChildField(final String fieldName) { + return findChildField(fieldName.hashCode(), fieldName); + } + + @Override + public FieldDescription findChildField(final int fieldNameHashCode, final String fieldName) { + for (final FieldDescription child : children) { + final String name = child.getFieldName(); + if (name == fieldName) { //NOSONAR NOPMD early return if the same String object reference + return child; + } + if (child.getFieldNameHashCode() == fieldNameHashCode && name.equals(fieldName)) { + return child; + } + } + return null; + } + + /** + * @return generic type argument name of the class (e.g. for List<String> this would return + * 'java.lang.String') + */ + public List getActualTypeArgumentNames() { + if (genericTypeNameList == null) { + genericTypeNameList = getActualTypeArguments().stream().map(Type::getTypeName).collect(Collectors.toList()); + } + + return genericTypeNameList; + } + + /** + * @return generic type argument objects of the class (e.g. for List<String> this would return 'String.class') + */ + public List getActualTypeArguments() { + if (genericTypeList == null) { + genericTypeList = new ArrayList<>(); + if ((fieldAccess == null) || (getGenericType() == null) || !(getGenericType() instanceof ParameterizedType)) { + return genericTypeList; + } + genericTypeList.addAll(Arrays.asList(((ParameterizedType) getGenericType()).getActualTypeArguments())); + } + + return genericTypeList; + } + + /** + * @return the children (if any) from the super classes + */ + @Override + public List getChildren() { + return children; + } + + @Override + public int getDataSize() { + return 0; + } + + @Override + public int getDataStartOffset() { + return 0; + } + + @Override + public int getDataStartPosition() { + return 0; + } + + /** + * @return the DataType (if known) for the detected Field, {@link DataType#OTHER} in all other cases + */ + @Override + public DataType getDataType() { + return dataType; + } + + /** + * @return the underlying Field type or {@code null} if it's a root node + */ + public FieldAccess getField() { + return fieldAccess; + } + + @Override + public String getFieldDescription() { + return fieldDescription; + } + + @Override + public String getFieldDirection() { + return fieldDirection; + } + + @Override + public List getFieldGroups() { + return fieldGroups; + } + + /** + * @return the underlying field name + */ + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public int getFieldNameHashCode() { + return fieldNameHashCode; + } + + /** + * @return relative field name within class hierarchy (ie. field_level0.field_level1.variable_0) + */ + public String getFieldNameRelative() { + return fieldNameRelative; + } + + public FieldSerialiser getFieldSerialiser() { + return fieldSerialiser; + } + + @Override + public int getFieldStart() { + throw new UnsupportedOperationException("not implemented"); + } + + @Override + public String getFieldUnit() { + return fieldUnit; + } + + /** + * @return field type strings (e.g. for the class Map<Integer,String> this returns + * '<java.lang.Integer,java.lang.String>' + */ + public String getGenericFieldTypeString() { + if (genericTypeNames == null) { + if (getActualTypeArgumentNames().isEmpty()) { + genericTypeNames = ""; + } else { + genericTypeNames = getActualTypeArgumentNames().stream().collect(Collectors.joining(", ", "<", ">")).intern(); + } + } + return genericTypeNames; + } + + /** + * @return field type strings (e.g. for the class Map<Integer,String> this returns + * '<java.lang.Integer,java.lang.String>' + */ + public String getGenericFieldTypeStringSimple() { + if (genericTypeNamesSimple == null) { + if (getActualTypeArgumentNames().isEmpty()) { + genericTypeNamesSimple = ""; + } else { + genericTypeNamesSimple = getActualTypeArguments().stream() // + .map(t -> ClassUtils.translateClassName(t.getTypeName())) + .collect(Collectors.joining(", ", "<", ">")) + .intern(); + } + } + return genericTypeNamesSimple; + } + + public Type getGenericType() { + if (genericType == null) { + genericType = fieldAccess == null ? new Type() { + @Override + public String getTypeName() { + return "unknown type"; + } + } : fieldAccess.field.getGenericType(); + } + return genericType; + } + + /** + * @return hierarchy level depth w.r.t. root object (ie. '0' being a variable in the root object) + */ + public int getHierarchyDepth() { + return hierarchyDepth; + } + + /** + * @return the modifierID + */ + public int getModifierID() { + return modifierID; + } + + /** + * @return the full modifier string (cached) + */ + public String getModifierString() { + if (modifierStr == null) { + // initialise only on a need to basis + // for performance reasons + modifierStr = Modifier.toString(modifierID).intern(); + } + return modifierStr; + } + + /** + * @return the parent + */ + @Override + public FieldDescription getParent() { + return parent; + } + + /** + * @param field class Field description for which + * @param hierarchyLevel the recursion level of the parent (e.g. '1' yields the immediate parent, '2' the parent of + * the parent etc.) + * @return the parent field reference description for the provided field + */ + public FieldDescription getParent(final FieldDescription field, final int hierarchyLevel) { + if (field == null) { + throw new IllegalArgumentException("field is null at hierarchyLevel = " + hierarchyLevel); + } + if ((hierarchyLevel == 0) || field.getParent() == null) { + return field; + } + return getParent(field.getParent(), hierarchyLevel - 1); + } + + /** + * @return field class type + */ + @Override + public Type getType() { + return classType; + } + + /** + * @return field class type name + */ + public String getTypeName() { + return typeName; + } + + /** + * @return field class type name + */ + public String getTypeNameSimple() { + return typeNameSimple; + } + + @Override + public int hashCode() { + return fieldNameHashCode; + } + + /** + * @return the isAbstract + */ + public boolean isAbstract() { + return modAbstract; + } + + @Override + public boolean isAnnotationPresent() { + return annotationPresent; + } + + /** + * @return the isClass + */ + public boolean isClass() { + return isClassType; + } + + /** + * @return whether class is an Enum type + */ + public boolean isEnum() { + return isEnumType; + } + + /** + * @return possible Enum definitions, see also 'isEnum()' + */ + public List getEnumConstants() { + return enumDefinitions; + } + + /** + * @return {@code true} if the class field includes the {@code final} modifier; {@code false} otherwise. + */ + public boolean isFinal() { + return modFinal; + } + + /** + * @return {@code true} if the class field is an interface + */ + public boolean isInterface() { + return modInterface; + } + + /** + * @return the isNative + */ + public boolean isNative() { + return modNative; + } + + /** + * @return {@code true} if the class field is a primitive type (ie. boolean, byte, ..., int, float, double) + */ + public boolean isPrimitive() { + return isPrimitiveType; + } + + /** + * @return {@code true} if the class field includes the {@code private} modifier; {@code false} otherwise. + */ + public boolean isPrivate() { + return modPrivate; + } + + /** + * @return the isProtected + */ + public boolean isProtected() { + return modProtected; + } + + /** + * @return {@code true} if the class field includes the {@code public} modifier; {@code false} otherwise. + */ + public boolean isPublic() { + return modPublic; + } + + /** + * @return the isRoot + */ + public boolean isRoot() { + return hierarchyDepth == 0; + } + + /** + * @return {@code true} if the class field should be serialised according to {@link java.io.Serializable} (ie. + * object's non-static and non-transient fields); {@code false} otherwise. + */ + public boolean isSerializable() { + return serializable; + } + + /** + * @return {@code true} if the class field includes the {@code static} modifier; {@code false} otherwise. + */ + public boolean isStatic() { + return modStatic; + } + + /** + * @return {@code true} if the class field includes the {@code strictfp} modifier; {@code false} otherwise. + */ + public boolean isStrict() { + return modStrict; + } + + /** + * @return {@code true} if the class field includes the {@code synchronized} modifier; {@code false} otherwise. + */ + public boolean isSynchronized() { + return modSynchronized; + } + + /** + * @return {@code true} if the class field includes the {@code transient} modifier; {@code false} otherwise. + */ + public boolean isTransient() { + return modTransient; + } + + /** + * @return {@code true} if the class field includes the {@code volatile} modifier; {@code false} otherwise. + */ + public boolean isVolatile() { + return modVolatile; + } + + @Override + public void printFieldStructure() { + printClassStructure(this, true, 0); + } + + public void setFieldSerialiser(final FieldSerialiser fieldSerialiser) { + this.fieldSerialiser = fieldSerialiser; + } + + @Override + public String toString() { + if (toStringName == null) { + toStringName = (ClassFieldDescription.class.getSimpleName() + " for: " + getModifierString() + " " + + getTypeName() + getGenericFieldTypeStringSimple() + " " + getFieldName() + " (hierarchyDepth = " + getHierarchyDepth() + ")") + .intern(); + } + return toStringName; + } + + protected static void exploreClass(final Class classType, final ClassFieldDescription parent, final int recursionLevel, final boolean fullScan) { // NOSONAR NOPMD + if (ClassUtils.DO_NOT_PARSE_MAP.get(classType) != null) { + return; + } + if (recursionLevel > ClassUtils.getMaxRecursionDepth()) { + throw new IllegalStateException("recursion error while scanning object structure: recursionLevel = '" + + recursionLevel + "' > " + ClassFieldDescription.class.getSimpleName() + ".maxRecursionLevel ='" + + ClassUtils.getMaxRecursionDepth() + "'"); + } + + // call super types + if ((classType.getSuperclass() != null) && !classType.getSuperclass().equals(Object.class) && !classType.getSuperclass().equals(Enum.class)) { + // dive into parent hierarchy w/o parsing Object.class, -> meaningless and causes infinite recursion + exploreClass(classType.getSuperclass(), parent, recursionLevel + 1, fullScan); + } + + // loop over member fields and inner classes + for (final Field pfield : classType.getDeclaredFields()) { + final FieldDescription localParent = parent.getParent(); + if ((localParent != null && pfield.getType().equals(localParent.getType()) && recursionLevel >= ClassUtils.getMaxRecursionDepth()) || pfield.getName().startsWith("this$")) { + // inner classes contain parent as part of declared fields + continue; + } + final ClassFieldDescription field = new ClassFieldDescription(pfield, parent, recursionLevel + 1, fullScan); // NOPMD + // N.B. unavoidable in-loop object generation + + // N.B. omitting field.isSerializable() (static or transient modifier) is essential + // as they often indicate class dependencies that are prone to infinite dependency loops + // (e.g. for classes with static references to themselves or maps-of-maps-of-maps-....) + final boolean isClassAndNotObjectOrEnmum = field.isClass() && (!field.getType().equals(Object.class) || !field.getType().equals(Enum.class)); + if (field.isSerializable() && (isClassAndNotObjectOrEnmum || field.isInterface()) && field.getDataType().equals(DataType.OTHER)) { + // object is a (technically) Serializable, unknown (ie 'OTHER) compound object or interface than can be further parsed + exploreClass(ClassUtils.getRawType(field.getType()), field, recursionLevel + 1, fullScan); + } + } + } + + protected static void printClassStructure(final ClassFieldDescription field, final boolean fullView, final int recursionLevel) { + final String enumOrClass = field.isEnum() ? "Enum " : "class "; + final String typeCategorgy = (field.isInterface() ? "interface " : (field.isPrimitive() ? "" : enumOrClass)); //NOSONAR //NOPMD + final String typeName = field.getTypeName() + field.getGenericFieldTypeString(); + final String mspace = spaces(recursionLevel * ClassUtils.getIndentationNumberOfSpace()); + final boolean isSerialisable = field.isSerializable(); + + if (isSerialisable || fullView) { + LOGGER.atInfo().addArgument(mspace).addArgument(isSerialisable ? " " : "//") // + .addArgument(field.getModifierString()) + .addArgument(typeCategorgy) + .addArgument(typeName) + .addArgument(field.getFieldName()) + .log("{} {} {} {}{} {}"); + if (field.isAnnotationPresent()) { + LOGGER.atInfo().addArgument(mspace).addArgument(isSerialisable ? " " : "//") // + .addArgument(field.getFieldUnit()) + .addArgument(field.getFieldDescription()) + .addArgument(field.getFieldDirection()) + .addArgument(field.getFieldGroups()) + .log("{} {} "); + } + + field.getChildren().forEach(f -> printClassStructure((ClassFieldDescription) f, fullView, recursionLevel + 1)); + } + } + + private static String getFieldDescription(final AnnotatedElement annotatedElement) { + final MetaInfo[] annotationMeta = annotatedElement.getAnnotationsByType(MetaInfo.class); + if (annotationMeta != null && annotationMeta.length > 0) { + return annotationMeta[0].description().intern(); + } + final Description[] annotationDescription = annotatedElement.getAnnotationsByType(Description.class); + if (annotationDescription != null && annotationDescription.length > 0) { + return annotationDescription[0].value().intern(); + } + return null; + } + + private static String getFieldDirection(final AnnotatedElement annotatedElement) { + final MetaInfo[] annotationMeta = annotatedElement.getAnnotationsByType(MetaInfo.class); + if (annotationMeta != null && annotationMeta.length > 0) { + return annotationMeta[0].direction().intern(); + } + final Direction[] annotationDirection = annotatedElement.getAnnotationsByType(Direction.class); + if (annotationDirection != null && annotationDirection.length > 0) { + return annotationDirection[0].value().intern(); + } + return null; + } + + private static List getFieldGroups(final AnnotatedElement annotatedElement) { + final MetaInfo[] annotationMeta = annotatedElement.getAnnotationsByType(MetaInfo.class); + if (annotationMeta != null && annotationMeta.length > 0) { + return Arrays.asList(annotationMeta[0].groups()); + } + final Groups[] annotationGroups = annotatedElement.getAnnotationsByType(Groups.class); + if (annotationGroups != null && annotationGroups.length > 0) { + final List ret = new ArrayList<>(annotationGroups[0].value().length); + for (int i = 0; i < annotationGroups[0].value().length; i++) { + ret.add(annotationGroups[0].value()[i].intern()); + } + return ret; + } + return Collections.emptyList(); + } + + private static String getFieldUnit(final AnnotatedElement annotatedElement) { + final MetaInfo[] annotationMeta = annotatedElement.getAnnotationsByType(MetaInfo.class); + if (annotationMeta != null && annotationMeta.length > 0) { + return annotationMeta[0].unit().intern(); + } + final Unit[] annotationUnit = annotatedElement.getAnnotationsByType(Unit.class); + if (annotationUnit != null && annotationUnit.length > 0) { + return annotationUnit[0].value().intern(); + } + return null; + } + + private static String spaces(final int spaces) { + return CharBuffer.allocate(spaces).toString().replace('\0', ' '); + } + + public static class FieldAccess { + private static final Unsafe unsafe; // NOPMD + + static { + // get an instance of the otherwise private 'Unsafe' class + try { + final Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); // NOSONAR NOPMD + unsafe = (Unsafe) field.get(null); + } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { // NOPMD + throw new SecurityException(e); // NOPMD + } + } + + private final Field field; + private final long fieldByteOffset; + + private FieldAccess(final Field field) { + this.field = field; + field.setAccessible(true); //NOSONAR + + long offset = -1; + try { + offset = unsafe.objectFieldOffset(field); + } catch (IllegalArgumentException e) { + // fails for private static final fields + } + this.fieldByteOffset = offset; + } + + public Object get(final Object classReference) { + return unsafe.getObject(classReference, fieldByteOffset); + } + + public boolean getBoolean(final Object classReference) { + return unsafe.getBoolean(classReference, fieldByteOffset); + } + + public byte getByte(final Object classReference) { + return unsafe.getByte(classReference, fieldByteOffset); + } + + public char getChar(final Object classReference) { + return unsafe.getChar(classReference, fieldByteOffset); + } + + public double getDouble(final Object classReference) { + return unsafe.getDouble(classReference, fieldByteOffset); + } + + public Field getField() { + return field; + } + public float getFloat(final Object classReference) { + return unsafe.getFloat(classReference, fieldByteOffset); + } + public int getInt(final Object classReference) { + return unsafe.getInt(classReference, fieldByteOffset); + } + public long getLong(final Object classReference) { + return unsafe.getLong(classReference, fieldByteOffset); + } + public short getShort(final Object classReference) { // NOPMD + return unsafe.getShort(classReference, fieldByteOffset); + } + public void set(final Object classReference, final Object obj) { + unsafe.putObject(classReference, fieldByteOffset, obj); + } + public void setBoolean(final Object classReference, final boolean value) { + unsafe.putBoolean(classReference, fieldByteOffset, value); + } + public void setByte(final Object classReference, final byte value) { + unsafe.putByte(classReference, fieldByteOffset, value); + } + public void setChar(final Object classReference, final char value) { + unsafe.putChar(classReference, fieldByteOffset, value); + } + + public void setDouble(final Object classReference, final double value) { + unsafe.putDouble(classReference, fieldByteOffset, value); + } + + public void setFloat(final Object classReference, final float value) { + unsafe.putFloat(classReference, fieldByteOffset, value); + } + + public void setInt(final Object classReference, final int value) { + unsafe.putInt(classReference, fieldByteOffset, value); + } + + public void setLong(final Object classReference, final long value) { + unsafe.putLong(classReference, fieldByteOffset, value); + } + + public void setShort(final Object classReference, final short value) { // NOPMD + unsafe.putShort(classReference, fieldByteOffset, value); + } + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/CmwLightSerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/CmwLightSerialiser.java new file mode 100644 index 00000000..58400e97 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/CmwLightSerialiser.java @@ -0,0 +1,1135 @@ +package io.opencmw.serialiser.spi; + +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.function.BiFunction; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.utils.ClassUtils; + +/** + * Light-weight open-source implementation of a (de-)serialiser that is binary-compatible to the serialiser used by CMW, + * a proprietary closed-source middle-ware used in some accelerator laboratories. + * + * N.B. this implementation is intended only for performance/functionality comparison and to enable a backward compatible + * transition to the {@link BinarySerialiser} implementation which is a bit more flexible, + * has some additional (optional) features, and a better IO performance. See the corresponding benchmarks for details; + * + * @author rstein + */ +@SuppressWarnings({ "PMD.ExcessiveClassLength", "PMD.ExcessivePublicCount", "PMD.TooManyMethods" }) +public class CmwLightSerialiser implements IoSerialiser { + public static final String NOT_IMPLEMENTED = "not implemented"; + private static final Logger LOGGER = LoggerFactory.getLogger(CmwLightSerialiser.class); + private static final int ADDITIONAL_HEADER_INFO_SIZE = 1000; + private static final DataType[] BYTE_TO_DATA_TYPE = new DataType[256]; + private static final Byte[] DATA_TYPE_TO_BYTE = new Byte[256]; + + static { + // static mapping of protocol bytes -- needed to be compatible with other wire protocols + // N.B. CmwLightSerialiser does not implement mappings for: + // * discreteFunction + // * discreteFunction_list + // * array of Data objects (N.B. 'Data' and nested 'Data' is being explicitely supported) + // 'Data' object is mapped to START_MARKER also used for nested data structures + + BYTE_TO_DATA_TYPE[0] = DataType.BOOL; + BYTE_TO_DATA_TYPE[1] = DataType.BYTE; + BYTE_TO_DATA_TYPE[2] = DataType.SHORT; + BYTE_TO_DATA_TYPE[3] = DataType.INT; + BYTE_TO_DATA_TYPE[4] = DataType.LONG; + BYTE_TO_DATA_TYPE[5] = DataType.FLOAT; + BYTE_TO_DATA_TYPE[6] = DataType.DOUBLE; + BYTE_TO_DATA_TYPE[201] = DataType.CHAR; // not actually implemented by CMW + BYTE_TO_DATA_TYPE[7] = DataType.STRING; + BYTE_TO_DATA_TYPE[8] = DataType.START_MARKER; // mapped to CMW 'Data' type + + // needs to be defined last + BYTE_TO_DATA_TYPE[9] = DataType.BOOL_ARRAY; + BYTE_TO_DATA_TYPE[10] = DataType.BYTE_ARRAY; + BYTE_TO_DATA_TYPE[11] = DataType.SHORT_ARRAY; + BYTE_TO_DATA_TYPE[12] = DataType.INT_ARRAY; + BYTE_TO_DATA_TYPE[13] = DataType.LONG_ARRAY; + BYTE_TO_DATA_TYPE[14] = DataType.FLOAT_ARRAY; + BYTE_TO_DATA_TYPE[15] = DataType.DOUBLE_ARRAY; + BYTE_TO_DATA_TYPE[202] = DataType.CHAR_ARRAY; // not actually implemented by CMW + BYTE_TO_DATA_TYPE[16] = DataType.STRING_ARRAY; + + // CMW 2D arrays -- also mapped internally to byte arrays + BYTE_TO_DATA_TYPE[17] = DataType.BOOL_ARRAY; + BYTE_TO_DATA_TYPE[18] = DataType.BYTE_ARRAY; + BYTE_TO_DATA_TYPE[19] = DataType.SHORT_ARRAY; + BYTE_TO_DATA_TYPE[20] = DataType.INT_ARRAY; + BYTE_TO_DATA_TYPE[21] = DataType.LONG_ARRAY; + BYTE_TO_DATA_TYPE[22] = DataType.FLOAT_ARRAY; + BYTE_TO_DATA_TYPE[23] = DataType.DOUBLE_ARRAY; + BYTE_TO_DATA_TYPE[203] = DataType.CHAR_ARRAY; // not actually implemented by CMW + BYTE_TO_DATA_TYPE[24] = DataType.STRING_ARRAY; + + // CMW multi-dim arrays -- also mapped internally to byte arrays + BYTE_TO_DATA_TYPE[25] = DataType.BOOL_ARRAY; + BYTE_TO_DATA_TYPE[26] = DataType.BYTE_ARRAY; + BYTE_TO_DATA_TYPE[27] = DataType.SHORT_ARRAY; + BYTE_TO_DATA_TYPE[28] = DataType.INT_ARRAY; + BYTE_TO_DATA_TYPE[29] = DataType.LONG_ARRAY; + BYTE_TO_DATA_TYPE[30] = DataType.FLOAT_ARRAY; + BYTE_TO_DATA_TYPE[31] = DataType.DOUBLE_ARRAY; + BYTE_TO_DATA_TYPE[204] = DataType.CHAR_ARRAY; // not actually implemented by CMW + BYTE_TO_DATA_TYPE[32] = DataType.STRING_ARRAY; + + for (int i = BYTE_TO_DATA_TYPE.length - 1; i >= 0; i--) { + if (BYTE_TO_DATA_TYPE[i] == null) { + continue; + } + final int id = BYTE_TO_DATA_TYPE[i].getID(); + DATA_TYPE_TO_BYTE[id] = (byte) i; + } + } + + private IoBuffer buffer; + private WireDataFieldDescription parent; + private WireDataFieldDescription lastFieldHeader; + private BiFunction> fieldSerialiserLookupFunction; + + public CmwLightSerialiser(final IoBuffer buffer) { + super(); + this.buffer = buffer; + } + + @Override + public ProtocolInfo checkHeaderInfo() { + final String fieldName = ""; + final int dataSize = FastByteBuffer.SIZE_OF_INT; + final WireDataFieldDescription headerStartField = new WireDataFieldDescription(this, parent, fieldName.hashCode(), fieldName, DataType.START_MARKER, buffer.position(), buffer.position(), dataSize); // NOPMD - needs to be read here + final int nEntries = buffer.getInt(); + if (nEntries <= 0) { + throw new IllegalStateException("nEntries = " + nEntries + " <= 0!"); + } + parent = lastFieldHeader = headerStartField; + return new ProtocolInfo(this, headerStartField, CmwLightSerialiser.class.getCanonicalName(), (byte) 1, (byte) 0, (byte) 1); + } + + @Override + public int[] getArraySizeDescriptor() { + final int nDims = buffer.getInt(); // number of dimensions + final int[] ret = new int[nDims]; + for (int i = 0; i < nDims; i++) { + ret[i] = buffer.getInt(); // vector size for each dimension + } + return ret; + } + + @Override + public boolean getBoolean() { + return buffer.getBoolean(); + } + + @Override + public boolean[] getBooleanArray(final boolean[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getBooleanArray(dst, length); + } + + @Override + public IoBuffer getBuffer() { + return buffer; + } + + @Override + public void setBuffer(final IoBuffer buffer) { + this.buffer = buffer; + } + + public int getBufferIncrements() { + return ADDITIONAL_HEADER_INFO_SIZE; + } + + @Override + public byte getByte() { + return buffer.getByte(); + } + + @Override + public byte[] getByteArray(final byte[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getByteArray(dst, length); + } + + @Override + public char getChar() { + return buffer.getChar(); + } + + @Override + public char[] getCharArray(final char[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getCharArray(dst, length); + } + + @Override + public Collection getCollection(final Collection collection) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public E getCustomData(final FieldSerialiser serialiser) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public double getDouble() { + return buffer.getDouble(); + } + + @Override + public double[] getDoubleArray(final double[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getDoubleArray(dst, length); + } + + @Override + public > Enum getEnum(final Enum enumeration) { + final int ordinal = buffer.getInt(); + assert ordinal >= 0 : "enum ordinal should be positive"; + + final String enumName = enumeration.getClass().getName(); + Class enumClass = ClassUtils.getClassByName(enumName); + if (enumClass == null) { + final String enumSimpleName = enumeration.getClass().getSimpleName(); + enumClass = ClassUtils.getClassByName(enumSimpleName); + if (enumClass == null) { + throw new IllegalStateException("could not find enum class description '" + enumName + "' or '" + enumSimpleName + "'"); + } + } + + try { + final Method values = enumClass.getMethod("values"); + final Object[] possibleEnumValues = (Object[]) values.invoke(null); + //noinspection unchecked + return (Enum) possibleEnumValues[ordinal]; // NOSONAR NOPMD + } catch (final ReflectiveOperationException e) { + LOGGER.atError().setCause(e).addArgument(enumClass).log("could not match 'valueOf(String)' function for class/(supposedly) enum of {}"); + } + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public String getEnumTypeList() { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public WireDataFieldDescription getFieldHeader() { + // process CMW-like wire-format + final int headerStart = buffer.position(); // NOPMD - need to read the present buffer position + + final String fieldName = buffer.getStringISO8859(); // NOPMD - read advances position + final byte dataTypeByte = buffer.getByte(); + final DataType dataType = getDataType(dataTypeByte); + // process CMW-like wire-format - done + + final int dataStartOffset = buffer.position() - headerStart; // NOPMD - further reads advance the read position in the buffer + final int dataSize; + if (dataType == DataType.START_MARKER) { + dataSize = FastByteBuffer.SIZE_OF_INT; + } else if (dataType.isScalar()) { + dataSize = dataType.getPrimitiveSize(); + } else if (dataType == DataType.STRING) { + dataSize = FastByteBuffer.SIZE_OF_INT + buffer.getInt(); // <(>string size -1> + + } else if (dataType.isArray() && dataType != DataType.STRING_ARRAY) { + // read array descriptor + final int[] dims = getArraySizeDescriptor(); + final int arraySize = buffer.getInt(); // strided array size + dataSize = FastByteBuffer.SIZE_OF_INT * (dims.length + 2) + arraySize * dataType.getPrimitiveSize(); // + + } else if (dataType == DataType.STRING_ARRAY) { + // read array descriptor -- this case has a high-penalty since the size of all Strings needs to be read + final int[] dims = getArraySizeDescriptor(); + final int arraySize = buffer.getInt(); // strided array size + // String parsing, need to follow every single element + int totalSize = FastByteBuffer.SIZE_OF_INT * arraySize; + for (int i = 0; i < arraySize; i++) { + final int stringSize = buffer.getInt(); // <(>string size -1> + + totalSize += stringSize; + buffer.position(buffer.position() + stringSize); + } + dataSize = FastByteBuffer.SIZE_OF_INT * (dims.length + 2) + totalSize; + } else { + throw new IllegalStateException("should not reach here -- format is incompatible with CMW"); + } + + final int fieldNameHashCode = fieldName.hashCode(); //TODO: verify same hashcode function + + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldNameHashCode, fieldName, dataType, headerStart, dataStartOffset, dataSize); + final int dataStartPosition = headerStart + dataStartOffset; + buffer.position(dataStartPosition); + + if (dataType == DataType.START_MARKER) { + parent = lastFieldHeader; + buffer.position(dataStartPosition); + buffer.position(dataStartPosition + dataSize); + } + + if (dataSize < 0) { + throw new IllegalStateException("should not reach here -- format is incompatible with CMW"); + } + return lastFieldHeader; + } + + @Override + public float getFloat() { + return buffer.getFloat(); + } + + @Override + public float[] getFloatArray(final float[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getFloatArray(dst, length); + } + + @Override + public int getInt() { + return buffer.getInt(); + } + + @Override + public int[] getIntArray(final int[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getIntArray(dst, length); + } + + @Override + public List getList(final List collection) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public long getLong() { + return buffer.getLong(); + } + + @Override + public long[] getLongArray(final long[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getLongArray(dst, length); + } + + @Override + public Map getMap(final Map map) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + public WireDataFieldDescription getParent() { + return parent; + } + + @Override + public Queue getQueue(final Queue collection) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public Set getSet(final Set collection) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public short getShort() { + return buffer.getShort(); + } + + @Override + public short[] getShortArray(final short[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getShortArray(dst, length); + } + + @Override + public String getString() { + return buffer.getString(); + } + + @Override + public String[] getStringArray(final String[] dst, final int length) { + getArraySizeDescriptor(); + return buffer.getStringArray(dst, length); + } + + @Override + public String getStringISO8859() { + return buffer.getStringISO8859(); + } + + /** + * @return {@code true} the ISO-8859-1 character encoding is being enforced for data fields (better performance), otherwise UTF-8 is being used (more generic encoding) + */ + public boolean isEnforceSimpleStringEncoding() { + return buffer.isEnforceSimpleStringEncoding(); + } + + /** + * + * @param state {@code true} the ISO-8859-1 character encoding is being enforced for data fields (better performance), otherwise UTF-8 is being used (more generic encoding) + */ + public void setEnforceSimpleStringEncoding(final boolean state) { + buffer.setEnforceSimpleStringEncoding(state); + } + + @Override + public boolean isPutFieldMetaData() { + return false; + } + + @Override + public void setPutFieldMetaData(final boolean putFieldMetaData) { + // do nothing -- not implemented for this serialiser + } + + @Override + public WireDataFieldDescription parseIoStream(final boolean readHeader) { + final WireDataFieldDescription fieldRoot = getRootElement(); + parent = fieldRoot; + final WireDataFieldDescription headerRoot = readHeader ? checkHeaderInfo().getFieldHeader() : getFieldHeader(); + buffer.position(headerRoot.getDataStartPosition() + headerRoot.getDataSize()); + parseIoStream(headerRoot, 0); + return fieldRoot; + } + + public void parseIoStream(final WireDataFieldDescription fieldRoot, final int recursionDepth) { + if (fieldRoot == null || fieldRoot.getDataType() != DataType.START_MARKER) { + throw new IllegalStateException("fieldRoot not a START_MARKER but: " + fieldRoot); + } + buffer.position(fieldRoot.getDataStartPosition()); + final int nEntries = buffer.getInt(); + if (nEntries < 0) { + throw new IllegalStateException("nEntries = " + nEntries + " < 0!"); + } + parent = lastFieldHeader = fieldRoot; + for (int i = 0; i < nEntries; i++) { + final WireDataFieldDescription field = getFieldHeader(); // NOPMD - need to read the present buffer position + final int dataSize = field.getDataSize(); + final int skipPosition = field.getDataStartPosition() + dataSize; // NOPMD - read at this location necessary, further reads advance position pointer + + if (field.getDataType() == DataType.START_MARKER) { + // detected sub-class start marker + parent = lastFieldHeader = field; + parseIoStream(field, recursionDepth + 1); + parent = lastFieldHeader = fieldRoot; + continue; + } + + if (dataSize < 0) { + LOGGER.atWarn().addArgument(field.getFieldName()).addArgument(field.getDataType()).addArgument(dataSize).log("WireDataFieldDescription for '{}' type '{}' has bytesToSkip '{} <= 0'"); + // fall-back option in case of undefined dataSetSize -- usually indicated an internal serialiser error + throw new IllegalStateException(); + } + + if (skipPosition < buffer.capacity()) { + buffer.position(skipPosition); + } else { + // reached end of buffer + if (skipPosition == buffer.capacity()) { + return; + } + throw new IllegalStateException("reached beyond end of buffer at " + skipPosition + " vs. capacity" + buffer.capacity() + " " + field); + } + } + } + + @Override + public void put(final FieldDescription fieldDescription, final Collection collection, final Type valueType) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public void put(final FieldDescription fieldDescription, final Enum enumeration) { + this.putFieldHeader(fieldDescription, DataType.INT); + buffer.putInt(enumeration.ordinal()); + } + + @Override + public void put(final FieldDescription fieldDescription, final Map map, Type keyType, Type valueType) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean value) { + this.putFieldHeader(fieldDescription, DataType.BOOL); + buffer.putBoolean(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.BOOL_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.BOOL_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte value) { + this.putFieldHeader(fieldDescription, DataType.BYTE); + buffer.putByte(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.BYTE_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.BYTE_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final char value) { + this.putFieldHeader(fieldDescription, DataType.CHAR); + buffer.putChar(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final char[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.CHAR_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final char[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.CHAR_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final double value) { + this.putFieldHeader(fieldDescription, DataType.DOUBLE); + buffer.putDouble(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final double[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.DOUBLE_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final double[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.DOUBLE_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final float value) { + this.putFieldHeader(fieldDescription, DataType.FLOAT); + buffer.putFloat(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final float[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.FLOAT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final float[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.FLOAT_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final int value) { + this.putFieldHeader(fieldDescription, DataType.INT); + buffer.putInt(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final int[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.INT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final int[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.INT_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final long value) { + this.putFieldHeader(fieldDescription, DataType.LONG); + buffer.putLong(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final long[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.LONG_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final long[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.LONG_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final short value) { // NOPMD by rstein + this.putFieldHeader(fieldDescription, DataType.SHORT); + buffer.putShort(value); + } + + @Override + public void put(final FieldDescription fieldDescription, final short[] values, final int n) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.SHORT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final short[] values, final int[] dims) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.SHORT_ARRAY); + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final String string) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.STRING); + buffer.putString(string); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final String[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.STRING_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final FieldDescription fieldDescription, final String[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldDescription, DataType.STRING_ARRAY); + final int nElements = putArraySizeDescriptor(dims); + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final boolean value) { + this.putFieldHeader(fieldName, DataType.BOOL); + buffer.putBoolean(value); + } + + @Override + public void put(final String fieldName, final boolean[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BOOL_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final boolean[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BOOL_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 17); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 25); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putBooleanArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final byte value) { + this.putFieldHeader(fieldName, DataType.BYTE); + buffer.putByte(value); + } + + @Override + public void put(final String fieldName, final byte[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BYTE_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final byte[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.BYTE_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 18); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 26); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putByteArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final char value) { + this.putFieldHeader(fieldName, DataType.CHAR); + buffer.putChar(value); + } + + @Override + public void put(final String fieldName, final char[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.CHAR_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final char[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.CHAR_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 203); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 204); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putCharArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final double value) { + this.putFieldHeader(fieldName, DataType.DOUBLE); + buffer.putDouble(value); + } + + @Override + public void put(final String fieldName, final double[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.DOUBLE_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final double[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.DOUBLE_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 23); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 31); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putDoubleArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final float value) { + this.putFieldHeader(fieldName, DataType.FLOAT); + buffer.putFloat(value); + } + + @Override + public void put(final String fieldName, final float[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.FLOAT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final float[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.FLOAT_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 22); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 30); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putFloatArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final int value) { + this.putFieldHeader(fieldName, DataType.INT); + buffer.putInt(value); + } + + @Override + public void put(final String fieldName, final int[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.INT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final int[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.INT_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 20); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 28); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putIntArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final long value) { + this.putFieldHeader(fieldName, DataType.LONG); + buffer.putLong(value); + } + + @Override + public void put(final String fieldName, final long[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.LONG_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final long[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.LONG_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 21); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 29); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putLongArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final short value) { // NOPMD by rstein + this.putFieldHeader(fieldName, DataType.SHORT); + buffer.putShort(value); + } + + @Override + public void put(final String fieldName, final short[] values, final int n) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.SHORT_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int bytesToCopy = putArraySizeDescriptor(n >= 0 ? Math.min(n, valuesSize) : valuesSize); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final short[] values, final int[] dims) { // NOPMD by rstein + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.SHORT_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 19); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 27); + } + final int bytesToCopy = putArraySizeDescriptor(dims); + buffer.putShortArray(values, bytesToCopy); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final String string) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.STRING); + buffer.putString(string); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final String[] values, final int n) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.STRING_ARRAY); + final int valuesSize = values == null ? 0 : values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + putArraySizeDescriptor(nElements); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final String[] values, final int[] dims) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.STRING_ARRAY); + if (dims.length == 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 24); + } else if (dims.length > 2) { + buffer.putByte(fieldHeader.getDataStartPosition() - 1, (byte) 32); + } + final int nElements = putArraySizeDescriptor(dims); + buffer.putStringArray(values, nElements); + updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final Collection collection, final Type valueType) { + final DataType dataType; + if (collection instanceof Queue) { + dataType = DataType.QUEUE; + } else if (collection instanceof Set) { + dataType = DataType.SET; + } else if (collection instanceof List) { + dataType = DataType.LIST; + } else { + dataType = DataType.COLLECTION; + } + + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, dataType); + this.put((FieldDescription) null, collection, valueType); + this.updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final Enum enumeration) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.ENUM); + this.put((FieldDescription) null, enumeration); + this.updateDataEndMarker(fieldHeader); + } + + @Override + public void put(final String fieldName, final Map map, final Type keyType, final Type valueType) { + final WireDataFieldDescription fieldHeader = putFieldHeader(fieldName, DataType.MAP); + this.put((FieldDescription) null, map, keyType, valueType); + this.updateDataEndMarker(fieldHeader); + } + + @Override + public int putArraySizeDescriptor(final int n) { + buffer.putInt(1); // number of dimensions + buffer.putInt(n); // vector size for each dimension + return n; + } + + @Override + public int putArraySizeDescriptor(final int[] dims) { + buffer.putInt(dims.length); // number of dimensions + int nElements = 1; + for (final int dim : dims) { + nElements *= dim; + buffer.putInt(dim); // vector size for each dimension + } + return nElements; + } + + @Override + public WireDataFieldDescription putCustomData(final FieldDescription fieldDescription, final E rootObject, Class type, final FieldSerialiser serialiser) { + throw new UnsupportedOperationException(NOT_IMPLEMENTED); + } + + @Override + public void putEndMarker(final FieldDescription fieldDescription) { + if (parent.getParent() != null) { + parent = (WireDataFieldDescription) parent.getParent(); + } + } + + @Override + public WireDataFieldDescription putFieldHeader(final FieldDescription fieldDescription) { + return putFieldHeader(fieldDescription, fieldDescription.getDataType()); + } + + public WireDataFieldDescription putFieldHeader(final FieldDescription fieldDescription, DataType customDataType) { + final boolean isScalar = customDataType.isScalar(); + + final int headerStart = buffer.position(); + final String fieldName = fieldDescription.getFieldName(); + buffer.putStringISO8859(fieldName); // full field name + buffer.putByte(getDataType(customDataType)); // data type ID + + final int dataStart = buffer.position(); + final int dataStartOffset = dataStart - headerStart; + final int dataSize; + if (isScalar) { + dataSize = customDataType.getPrimitiveSize(); + } else if (customDataType == DataType.START_MARKER) { + dataSize = FastByteBuffer.SIZE_OF_INT; + buffer.ensureAdditionalCapacity(dataSize); + } else { + dataSize = -1; + // from hereon there are data specific structures + buffer.ensureAdditionalCapacity(16); // allocate 16+ bytes to account for potential array header (safe-bet) + } + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldDescription.getFieldNameHashCode(), fieldDescription.getFieldName(), customDataType, headerStart, dataStartOffset, dataSize); + updateDataEntryCount(); + + return lastFieldHeader; + } + + @Override + public WireDataFieldDescription putFieldHeader(final String fieldName, final DataType dataType) { + final boolean isScalar = dataType.isScalar(); + + final int headerStart = buffer.position(); + buffer.putStringISO8859(fieldName); // full field name + buffer.putByte(getDataType(dataType)); // data type ID + + final int dataStart = buffer.position(); + final int dataStartOffset = dataStart - headerStart; + final int dataSize; + if (isScalar) { + dataSize = dataType.getPrimitiveSize(); + } else if (dataType == DataType.START_MARKER) { + dataSize = FastByteBuffer.SIZE_OF_INT; + buffer.ensureAdditionalCapacity(dataSize); + } else { + dataSize = -1; + // from hereon there are data specific structures + buffer.ensureAdditionalCapacity(16); // allocate 16+ bytes to account for potential array header (safe-bet) + } + + final int fieldNameHashCode = fieldName.hashCode(); // TODO: check hashCode function + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldNameHashCode, fieldName, dataType, headerStart, dataStartOffset, dataSize); + updateDataEntryCount(); + + return lastFieldHeader; + } + + @Override + public void putHeaderInfo(final FieldDescription... field) { + parent = lastFieldHeader = getRootElement(); + final String fieldName = ""; + final int dataSize = FastByteBuffer.SIZE_OF_INT; + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldName.hashCode(), fieldName, DataType.START_MARKER, buffer.position(), buffer.position(), dataSize); + buffer.putInt(0); + updateDataEntryCount(); + parent = lastFieldHeader; + } + + @Override + public void putStartMarker(final FieldDescription fieldDescription) { + putFieldHeader(fieldDescription, DataType.START_MARKER); + buffer.putInt(0); + updateDataEndMarker(lastFieldHeader); + parent = lastFieldHeader; + } + + @Override + public void setQueryFieldName(final String fieldName, final int dataStartPosition) { + if (fieldName == null || fieldName.isBlank()) { + throw new IllegalArgumentException("fieldName must not be null or blank: " + fieldName); + } + buffer.position(dataStartPosition); + } + + @Override + public void updateDataEndMarker(final WireDataFieldDescription fieldHeader) { + final int dataSize = buffer.position() - fieldHeader.getDataStartPosition(); + if (fieldHeader.getDataSize() != dataSize) { + fieldHeader.setDataSize(dataSize); + } + } + + private WireDataFieldDescription getRootElement() { + return new WireDataFieldDescription(this, null, "ROOT".hashCode(), "ROOT", DataType.OTHER, buffer.position(), -1, -1); + } + + private void updateDataEntryCount() { + // increment parent child count + if (parent == null) { + throw new IllegalStateException("no parent"); + } + + final int parentDataStart = parent.getDataStartPosition(); + if (parentDataStart >= 0) { // N.B. needs to be '>=' since CMW header is an incomplete field header containing only an 'nEntries' data field + buffer.position(parentDataStart); + final int nEntries = buffer.getInt(); + buffer.position(parentDataStart); + buffer.putInt(nEntries + 1); + buffer.position(lastFieldHeader.getDataStartPosition()); + } + } + + public static byte getDataType(final DataType dataType) { + final int id = dataType.getID(); + if (DATA_TYPE_TO_BYTE[id] != null) { + return DATA_TYPE_TO_BYTE[id]; + } + + throw new IllegalArgumentException("DataType " + dataType + " not mapped to specific byte"); + } + + public static DataType getDataType(final byte byteValue) { + final int id = byteValue & 0xFF; + if (BYTE_TO_DATA_TYPE[id] != null) { + return BYTE_TO_DATA_TYPE[id]; + } + + throw new IllegalArgumentException("DataType byteValue=" + byteValue + " rawByteValue=" + (byteValue & 0xFF) + " not mapped"); + } + + @Override + public void setFieldSerialiserLookupFunction(final BiFunction> serialiserLookupFunction) { + this.fieldSerialiserLookupFunction = serialiserLookupFunction; + } + + @Override + public BiFunction> getSerialiserLookupFunction() { + return fieldSerialiserLookupFunction; + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/FastByteBuffer.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/FastByteBuffer.java new file mode 100644 index 00000000..2365ab28 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/FastByteBuffer.java @@ -0,0 +1,1088 @@ +package io.opencmw.serialiser.spi; + +import static sun.misc.Unsafe.*; // NOSONAR NOPMD not an issue: contained and performance-related use + +import java.lang.reflect.Field; +import java.util.Objects; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.utils.AssertUtils; +import io.opencmw.serialiser.utils.ByteArrayCache; + +import sun.misc.Unsafe; // NOPMD - there is still nothing better under the Sun + +// import static jdk.internal.misc.Unsafe; // NOPMD by rstein TODO replaces sun in JDK11 + +/** + * FastByteBuffer implementation based on JVM 'Unsafe' Class. based on: + * https://mechanical-sympathy.blogspot.com/2012/07/native-cc-like-performance-for-java.html + * http://java-performance.info/various-methods-of-binary-serialization-in-java/ + * + * All accesses are range checked, because the performance impact was determined to be negligible. + * + * Read operations return "IndexOutOfBoundsException" if there are not enough bytes left in the buffer. + * For primitive types, the check can be done before, but for arrays and strings the size field has to be read first. + * Therefore, the position after a failed non-primitive read is not necessarily the position before the read attempt. + * + * When there is not enough space for a write operation, the behaviour depends on the autoRange and byteArrayCache + * variables. If autoRange is false, the operation returns an IndexOutOfBounds exception and the position is set to the + * position before the operation. For Strings there is a worst case space estimate being done, so an operation might + * fail with enough space left. If autoRange is true, the underlying byte array is replaced by a bigger one which is at + * least 1 KiB byte or 12,5% , but at max 100 KiB bigger than the requested size. When a byteArrayCache is provided, the + * next biggest array in that cache is used and the old buffer is returned, otherwise a new byte[] array is allocated. + * + * @author rstein + */ +@SuppressWarnings({ "restriction", "PMD.TooManyMethods", "PMD.ExcessivePublicCount", "PMD.ExcessiveClassLength" }) // unavoidable: each primitive type needs to handled individually (no templates) +public class FastByteBuffer implements IoBuffer { + public static final int SIZE_OF_BOOLEAN = 1; + public static final int SIZE_OF_BYTE = 1; + public static final int SIZE_OF_SHORT = 2; + public static final int SIZE_OF_CHAR = 2; + public static final int SIZE_OF_INT = 4; + public static final int SIZE_OF_LONG = 8; + public static final int SIZE_OF_FLOAT = 4; + public static final int SIZE_OF_DOUBLE = 8; + public static final String INVALID_UTF_8 = "Invalid UTF-8"; + private static final int DEFAULT_INITIAL_CAPACITY = 1 << 10; + private static final int DEFAULT_MIN_CAPACITY_INCREASE = 1 << 10; + private static final int DEFAULT_MAX_CAPACITY_INCREASE = 100 * (1 << 10); + private static final Unsafe unsafe; // NOPMD + static { + // get an instance of the otherwise private 'Unsafe' class + try { + @SuppressWarnings("Java9ReflectionClassVisibility") + Class cls = Class.forName("jdk.internal.module.IllegalAccessLogger"); // NOSONAR NOPMD + Field logger = cls.getDeclaredField("logger"); + + final Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); //NOSONAR + unsafe = (Unsafe) field.get(null); + unsafe.putObjectVolatile(cls, unsafe.staticFieldOffset(logger), null); + + } catch (NoSuchFieldException | SecurityException | IllegalAccessException | ClassNotFoundException e) { // NOPMD + throw new SecurityException(e); // NOPMD + } + } + + private final ReadWriteLock internalLock = new ReentrantReadWriteLock(); + private final StringBuilder builder = new StringBuilder(100); // NOPMD + private ByteArrayCache byteArrayCache; + private int intPos; + private int intLimit; + private byte[] buffer; + private boolean enforceSimpleStringEncoding; + private boolean autoResize; + + /** + * construct new FastByteBuffer backed by a default length array + */ + public FastByteBuffer() { + this(DEFAULT_INITIAL_CAPACITY); + } + /** + * construct new FastByteBuffer + * + * @param buffer buffer to initialise/re-use (stored directly) + * @param limit position until buffer is filled + */ + @SuppressWarnings("PMD.ArrayIsStoredDirectly") + public FastByteBuffer(final byte[] buffer, final int limit) { + Objects.requireNonNull(buffer, "buffer"); + if (buffer.length < limit) { + throw new IllegalArgumentException(String.format("limit %d >= capacity %d", limit, buffer.length)); + } + this.buffer = buffer; + this.intLimit = limit; + intPos = 0; + } + + /** + * construct new FastByteBuffer + * + * @param size initial capacity of the buffer + */ + public FastByteBuffer(final int size) { + this(size, false, null); + } + + /** + * construct new FastByteBuffer + * + * @param size initial capacity of the buffer + * @param autoResize whether the buffer should be resized automatically when trying to write past capacity + * @param byteArrayCache a ByteArrayCache from which arrays are obtained and returned to on resize + */ + public FastByteBuffer(final int size, final boolean autoResize, final ByteArrayCache byteArrayCache) { + AssertUtils.gtEqThanZero("size", size); + buffer = new byte[size]; + intPos = 0; + intLimit = buffer.length; + this.autoResize = autoResize; + this.byteArrayCache = byteArrayCache; + } + + @Override + public int capacity() { + return buffer.length; + } + + public void checkAvailable(final int bytes) { + if (intPos + bytes > intLimit) { + throw new IndexOutOfBoundsException("read unavailable " + bytes + " bytes at position " + intPos + " (limit: " + intLimit + ")"); + } + } + + public void checkAvailableAbsolute(final int position) { + if (position > intLimit) { + throw new IndexOutOfBoundsException("read unavailable bytes at end position " + position + " (limit: " + intLimit + ")"); + } + } + + @Override + public void clear() { + intPos = 0; + intLimit = capacity(); + } + + /** + * + * @return access to internal storage array (N.B. this is volatile and may be overwritten in case of auto-grow, or external sets) + */ + @Override + public byte[] elements() { + return buffer; // NOPMD -- allow public access to internal array + } + + @Override + public void ensureAdditionalCapacity(final int capacity) { + ensureCapacity(this.position() + capacity); + } + + @Override + public void ensureCapacity(final int newCapacity) { + if (newCapacity <= capacity()) { + return; + } + if (intPos > capacity()) { // invalid state, should never occur + throw new IllegalStateException("position " + intPos + " is beyond buffer capacity " + capacity()); + } + if (!autoResize) { + throw new IndexOutOfBoundsException("required capacity: " + newCapacity + " out of bounds: " + capacity() + "and autoResize is disabled"); + } + //TODO: add smarter enlarging algorithm (ie. increase fast for small arrays, + n% for medium sized arrays, byte-by-byte for large arrays) + final int addCapacity = Math.min(Math.max(DEFAULT_MIN_CAPACITY_INCREASE, newCapacity >> 3), DEFAULT_MAX_CAPACITY_INCREASE); // min, +12.5%, max + // if we are reading, limit() marks valid data, when writing, position() marks end of valid data, limit() is safe bet because position <= limit + forceCapacity(newCapacity + addCapacity, limit()); + } + + @Override + public void flip() { + intLimit = intPos; + intPos = 0; + } + + /** + * Forces FastByteBuffer to contain the given number of entries, preserving just a part of the array. + * + * @param length the new minimum length for this array. + * @param preserve the number of elements of the old buffer that shall be preserved in case a new allocation is necessary. + */ + @Override + public void forceCapacity(final int length, final int preserve) { + if (length == capacity()) { + intLimit = length; + return; + } + final byte[] newBuffer = byteArrayCache == null ? new byte[length] : byteArrayCache.getArray(length); + final int bytesToCopy = preserve * SIZE_OF_BYTE; + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET, newBuffer, ARRAY_BYTE_BASE_OFFSET, bytesToCopy); + intPos = Math.min(intPos, newBuffer.length); + if (byteArrayCache != null) { + byteArrayCache.add(buffer); + } + buffer = newBuffer; + intLimit = buffer.length; + } + + @Override + public boolean getBoolean() { // NOPMD by rstein + checkAvailable(SIZE_OF_BOOLEAN); + final boolean value = unsafe.getBoolean(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); + intPos += SIZE_OF_BOOLEAN; + + return value; + } + + @Override + public boolean getBoolean(final int position) { + checkAvailableAbsolute(position + SIZE_OF_BOOLEAN); + return unsafe.getBoolean(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public boolean[] getBooleanArray(final boolean[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final boolean[] values = initNeeded ? new boolean[arraySize] : dst; + + checkAvailable(arraySize * SIZE_OF_BOOLEAN); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_BOOLEAN_BASE_OFFSET, arraySize); + intPos += arraySize; + + return values; + } + + @Override + public byte getByte() { + checkAvailable(SIZE_OF_BYTE); + final byte value = unsafe.getByte(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); + intPos += SIZE_OF_BYTE; + + return value; + } + + @Override + public byte getByte(final int position) { + checkAvailableAbsolute(position + SIZE_OF_BYTE); + return unsafe.getByte(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public byte[] getByteArray(final byte[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final byte[] values = initNeeded ? new byte[arraySize] : dst; + + checkAvailable(arraySize); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_BYTE_BASE_OFFSET, arraySize); + intPos += arraySize; + + return values; + } + + public ByteArrayCache getByteArrayCache() { + return byteArrayCache; + } + + @Override + public char getChar() { + checkAvailable(SIZE_OF_CHAR); + final char value = unsafe.getChar(buffer, (long) ARRAY_CHAR_BASE_OFFSET + intPos); + intPos += SIZE_OF_CHAR; + + return value; + } + + @Override + public char getChar(final int position) { + checkAvailableAbsolute(position + SIZE_OF_CHAR); + return unsafe.getChar(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public char[] getCharArray(final char[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final char[] values = initNeeded ? new char[arraySize] : dst; + + final int bytesToCopy = arraySize * SIZE_OF_CHAR; + checkAvailable(bytesToCopy); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_SHORT_BASE_OFFSET, bytesToCopy); + intPos += bytesToCopy; + + return values; + } + + @Override + public double getDouble() { + checkAvailable(SIZE_OF_DOUBLE); + final double value = unsafe.getDouble(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); + intPos += SIZE_OF_DOUBLE; + + return value; + } + + @Override + public double getDouble(final int position) { + checkAvailableAbsolute(position + SIZE_OF_DOUBLE); + return unsafe.getDouble(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public double[] getDoubleArray(final double[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final double[] values = initNeeded ? new double[arraySize] : dst; + + final int bytesToCopy = arraySize * SIZE_OF_DOUBLE; + checkAvailable(bytesToCopy); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_DOUBLE_BASE_OFFSET, bytesToCopy); + intPos += bytesToCopy; + + return values; + } + + @Override + public float getFloat() { + checkAvailable(SIZE_OF_FLOAT); + final float value = unsafe.getFloat(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); + intPos += SIZE_OF_FLOAT; + + return value; + } + + @Override + public float getFloat(final int position) { + checkAvailableAbsolute(position + SIZE_OF_FLOAT); + return unsafe.getFloat(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public float[] getFloatArray(final float[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final float[] values = initNeeded ? new float[arraySize] : dst; + + final int bytesToCopy = arraySize * SIZE_OF_FLOAT; + checkAvailable(bytesToCopy); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_FLOAT_BASE_OFFSET, bytesToCopy); + intPos += bytesToCopy; + + return values; + } + + @Override + public int getInt() { + checkAvailable(SIZE_OF_INT); + final int value = unsafe.getInt(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); + intPos += SIZE_OF_INT; + + return value; + } + + @Override + public int getInt(final int position) { + checkAvailableAbsolute(position + SIZE_OF_INT); + return unsafe.getInt(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public int[] getIntArray(final int[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final int[] values = initNeeded ? new int[arraySize] : dst; + + final int bytesToCopy = arraySize * SIZE_OF_INT; + checkAvailable(bytesToCopy); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_INT_BASE_OFFSET, bytesToCopy); + intPos += bytesToCopy; + + return values; + } + + @Override + public long getLong() { + checkAvailable(SIZE_OF_LONG); + final long value = unsafe.getLong(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); + intPos += SIZE_OF_LONG; + + return value; + } + + @Override + public long getLong(final int position) { + checkAvailableAbsolute(position + SIZE_OF_LONG); + return unsafe.getLong(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public long[] getLongArray(final long[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final long[] values = initNeeded ? new long[arraySize] : dst; + + final int bytesToCopy = arraySize * SIZE_OF_LONG; + checkAvailable(bytesToCopy); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_LONG_BASE_OFFSET, bytesToCopy); + intPos += bytesToCopy; + + return values; + } + + @Override + public short getShort() { // NOPMD by rstein + checkAvailable(SIZE_OF_SHORT); + final short value = unsafe.getShort(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos); // NOPMD + intPos += SIZE_OF_SHORT; + + return value; + } + + @Override + public short getShort(final int position) { + checkAvailableAbsolute(position + SIZE_OF_SHORT); + return unsafe.getShort(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position); + } + + @Override + public short[] getShortArray(final short[] dst, final int length) { // NOPMD by rstein + final int arraySize = getInt(); // strided-array size + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final short[] values = initNeeded ? new short[arraySize] : dst; // NOPMD by rstein + + final int bytesToCopy = arraySize * SIZE_OF_SHORT; + checkAvailable(bytesToCopy); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET + intPos, values, ARRAY_SHORT_BASE_OFFSET, bytesToCopy); + intPos += bytesToCopy; + + return values; + } + + @Override + public String getString() { + if (isEnforceSimpleStringEncoding()) { + return this.getStringISO8859(); + } + final int arraySize = getInt(); // for C++ zero terminated string + checkAvailable(arraySize); + // alt: final String str = new String(buffer, position, arraySize - 1, StandardCharsets.UTF_8) + decodeUTF8(buffer, intPos, arraySize - 1, builder); + + intPos += arraySize; // N.B. +1 larger to be compatible with C++ zero terminated string + // alt: return str + return builder.toString(); + } + + @Override + public String getString(final int position) { + final int oldPosition = position(); + position(position); + final String ret = getString(); + position(oldPosition); + return ret; + } + + @Override + public String[] getStringArray(final String[] dst, final int length) { + final int arraySize = getInt(); // strided-array size + checkAvailable(arraySize); + final boolean initNeeded = dst == null || length < 0 || dst.length != arraySize; + final String[] ret = initNeeded ? new String[arraySize] : dst; + for (int k = 0; k < arraySize; k++) { + ret[k] = getString(); + } + return ret; + } + + @Override + public String getStringISO8859() { + final int arraySize = getInt(); // for C++ zero terminated string + checkAvailable(arraySize); + //alt safe-fallback final String str = new String(buffer, position, arraySize - 1, StandardCharsets.ISO_8859_1) + @SuppressWarnings("deprecation") + final String str = new String(buffer, 0, intPos, arraySize - 1); // NOSONAR NOPMD fastest alternative that is public API + // final String str = FastStringBuilder.iso8859BytesToString(buffer, position, arraySize - 1) + intPos += arraySize; // N.B. +1 larger to be compatible with C++ zero terminated string + return str; + } + + @Override + public boolean hasRemaining() { + return (this.position() < limit()); + } + + /** + * @return True if the underlying byte array will be replaced by a bigger one if there is not enough space left + */ + public boolean isAutoResize() { + return autoResize; + } + + @Override + public boolean isEnforceSimpleStringEncoding() { + return enforceSimpleStringEncoding; + } + + @Override + public boolean isReadOnly() { + return false; + } + + @Override + public int limit() { + return intLimit; + } + + @Override + public void limit(final int newLimit) { + if ((newLimit > capacity()) || (newLimit < 0)) { + throw new IllegalArgumentException(String.format("invalid newLimit: [0, position: %d, newLimit:%d, %d]", intPos, newLimit, capacity())); + } + intLimit = newLimit; + if (intPos > intLimit) { + intPos = intLimit; + } + } + + @Override + public ReadWriteLock lock() { + return internalLock; + } + + @Override + public int position() { + return intPos; + } + + @Override + public void position(final int newPosition) { + if ((newPosition > intLimit) || (newPosition < 0) || (newPosition > capacity())) { + throw new IllegalArgumentException(String.format("invalid newPosition: %d vs. [0, position=%d, limit:%d, capacity:%d]", newPosition, intPos, intLimit, capacity())); + } + intPos = newPosition; + } + + @Override + public void putBoolean(final boolean value) { + ensureAdditionalCapacity(SIZE_OF_BOOLEAN); + unsafe.putBoolean(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_BOOLEAN; + } + + @Override + public void putBoolean(final int position, final boolean value) { + ensureCapacity(position + SIZE_OF_BOOLEAN); + unsafe.putBoolean(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putBooleanArray(final boolean[] values, final int n) { + final int valuesSize = values == null ? 0 : values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + ensureAdditionalCapacity(nElements); + putInt(nElements); // strided-array size + copyMemory(values, ARRAY_BOOLEAN_BASE_OFFSET, buffer, ARRAY_BYTE_BASE_OFFSET + intPos, nElements); + intPos += nElements; + } + + @Override + public void putByte(final byte value) { + ensureAdditionalCapacity(SIZE_OF_BYTE); + unsafe.putByte(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_BYTE; + } + + @Override + public void putByte(final int position, final byte value) { + ensureCapacity(position + SIZE_OF_BYTE); + unsafe.putByte(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putByteArray(final byte[] values, final int n) { + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + ensureAdditionalCapacity(nElements + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, ARRAY_BOOLEAN_BASE_OFFSET, buffer, ARRAY_BYTE_BASE_OFFSET + intPos, nElements); + intPos += nElements; + } + + @Override + public void putChar(final char value) { + ensureAdditionalCapacity(SIZE_OF_CHAR); + unsafe.putChar(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_CHAR; + } + + @Override + public void putChar(final int position, final char value) { + ensureCapacity(position + SIZE_OF_CHAR); + unsafe.putChar(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putCharArray(final char[] values, final int n) { + final int arrayOffset = ARRAY_CHAR_BASE_OFFSET; + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + final int bytesToCopy = nElements * SIZE_OF_CHAR; + ensureAdditionalCapacity(bytesToCopy + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, arrayOffset, buffer, arrayOffset + intPos, bytesToCopy); + intPos += bytesToCopy; + } + + @Override + public void putDouble(final double value) { + ensureAdditionalCapacity(SIZE_OF_DOUBLE); + unsafe.putDouble(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_DOUBLE; + } + + @Override + public void putDouble(final int position, final double value) { + ensureCapacity(position + SIZE_OF_DOUBLE); + unsafe.putDouble(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putDoubleArray(final double[] values, final int n) { + final int arrayOffset = ARRAY_DOUBLE_BASE_OFFSET; + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + final int bytesToCopy = nElements * SIZE_OF_DOUBLE; + ensureAdditionalCapacity(bytesToCopy + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, arrayOffset, buffer, arrayOffset + intPos, bytesToCopy); + intPos += bytesToCopy; + } + + @Override + public void putFloat(final float value) { + ensureAdditionalCapacity(SIZE_OF_FLOAT); + unsafe.putFloat(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_FLOAT; + } + + @Override + public void putFloat(final int position, final float value) { + ensureCapacity(position + SIZE_OF_FLOAT); + unsafe.putFloat(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putFloatArray(final float[] values, final int n) { + final int arrayOffset = ARRAY_FLOAT_BASE_OFFSET; + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + final int bytesToCopy = nElements * SIZE_OF_FLOAT; + ensureAdditionalCapacity(bytesToCopy + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, arrayOffset, buffer, arrayOffset + intPos, bytesToCopy); + intPos += bytesToCopy; + } + + @Override + public void putInt(final int value) { + ensureAdditionalCapacity(SIZE_OF_INT); + unsafe.putInt(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_INT; + } + + @Override + public void putInt(final int position, final int value) { + ensureCapacity(position + SIZE_OF_INT); + unsafe.putInt(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putIntArray(final int[] values, final int n) { + final int arrayOffset = ARRAY_INT_BASE_OFFSET; + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + final int bytesToCopy = nElements * SIZE_OF_INT; + ensureAdditionalCapacity(bytesToCopy + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, arrayOffset, buffer, arrayOffset + intPos, bytesToCopy); + intPos += bytesToCopy; + } + + @Override + public void putLong(final long value) { + ensureAdditionalCapacity(SIZE_OF_LONG); + unsafe.putLong(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_LONG; + } + + @Override + public void putLong(final int position, final long value) { + ensureCapacity(position + SIZE_OF_LONG); + unsafe.putLong(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putLongArray(final long[] values, final int n) { + final int arrayOffset = ARRAY_LONG_BASE_OFFSET; + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + final int bytesToCopy = nElements * SIZE_OF_LONG; + ensureAdditionalCapacity(bytesToCopy + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, arrayOffset, buffer, arrayOffset + intPos, bytesToCopy); + intPos += bytesToCopy; + } + + @Override + public void putShort(final short value) { // NOPMD by rstein + ensureAdditionalCapacity(SIZE_OF_SHORT); + unsafe.putShort(buffer, (long) ARRAY_BYTE_BASE_OFFSET + intPos, value); + intPos += SIZE_OF_SHORT; + } + + @Override + public void putShort(final int position, final short value) { + ensureCapacity(position + SIZE_OF_SHORT); + unsafe.putShort(buffer, (long) ARRAY_BYTE_BASE_OFFSET + position, value); + } + + @Override + public void putShortArray(final short[] values, final int n) { // NOPMD by rstein + final int arrayOffset = ARRAY_SHORT_BASE_OFFSET; + final int valuesSize = values == null ? 0 : values.length; + final int nElements = (n >= 0 ? Math.min(n, valuesSize) : valuesSize); + final int bytesToCopy = nElements * SIZE_OF_SHORT; + ensureAdditionalCapacity(bytesToCopy + SIZE_OF_INT); + putInt(nElements); // strided-array size + copyMemory(values, arrayOffset, buffer, arrayOffset + intPos, bytesToCopy); + intPos += bytesToCopy; + } + + @Override + public void putString(final String string) { + if (string == null) { + putString(""); + return; + } + if (isEnforceSimpleStringEncoding()) { + putStringISO8859(string); + return; + } + final int utf16StringLength = string.length(); + final int initialPos = intPos; + intPos += SIZE_OF_INT; + // write string-to-byte (in-place) + ensureAdditionalCapacity(3 * utf16StringLength + SIZE_OF_INT); + final int strLength = encodeUTF8(string, buffer, intPos, 3 * utf16StringLength); + final int endPos = intPos + strLength; + + // write length of string byte representation + putInt(initialPos, strLength + 1); + intPos = endPos; + + putByte((byte) 0); // For C++ zero terminated string + } + + @Override + public void putString(final int position, final String value) { + final int oldPosition = position(); + position(position); + putString(value); + position(oldPosition); + } + + @Override + public void putStringArray(final String[] values, final int n) { + final int valuesSize = values == null ? 0 : values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + final int originalPos = intPos; // NOPMD + putInt(nElements); // strided-array size + if (values == null) { + return; + } + try { + if (isEnforceSimpleStringEncoding()) { + for (int k = 0; k < nElements; k++) { + putStringISO8859(values[k]); + } + return; + } + for (int k = 0; k < nElements; k++) { + putString(values[k]); + } + } catch (IndexOutOfBoundsException e) { + intPos = originalPos; // reset the position to the original position before any strings where added + throw e; // rethrow the exception + } + } + + @Override + public void putStringISO8859(final String string) { + if (string == null) { + putStringISO8859(""); + return; + } + final int initialPos = intPos; + intPos += SIZE_OF_INT; + // write string-to-byte (in-place) + final int strLength = encodeISO8859(string, buffer, intPos, string.length()); + final int endPos = intPos + strLength; + + // write length of string byte representation + intPos = initialPos; + putInt(strLength + 1); + intPos = endPos; + + putByte((byte) 0); // For C++ zero terminated string + } + + @Override + public int remaining() { + return intLimit - intPos; + } + + @Override + public void reset() { + intPos = 0; + intLimit = buffer.length; + } + + /** + * @param autoResize {@code true} to automatically increase the size of the underlying buffer if necessary + */ + public void setAutoResize(final boolean autoResize) { + this.autoResize = autoResize; + } + + public void setByteArrayCache(final ByteArrayCache byteArrayCache) { + this.byteArrayCache = byteArrayCache; + } + + @Override + public void setEnforceSimpleStringEncoding(final boolean state) { + this.enforceSimpleStringEncoding = state; + } + + @Override + public String toString() { + return super.toString() + String.format(" - [0, position=%d, limit:%d, capacity:%d]", intPos, intLimit, capacity()); + } + + /** + * Trims the internal buffer array so that the capacity is equal to the size. + * + * @see java.util.ArrayList#trimToSize() + */ + @Override + public void trim() { + trim(position()); + } + + /** + * Trims the internal buffer array if it is too large. If the current array length is smaller than or equal to + * {@code requestedCapacity}, this method does nothing. Otherwise, it trims the array length to the maximum between + * {@code requestedCapacity} and {@link #capacity()}. + *

+ * This method is useful when reusing FastBuffers. {@linkplain #reset() Clearing a list} leaves the array length + * untouched. If you are reusing a list many times, you can call this method with a typical size to avoid keeping + * around a very large array just because of a few large transient lists. + * + * @param requestedCapacity the threshold for the trimming. + */ + @Override + public void trim(final int requestedCapacity) { + if ((requestedCapacity >= capacity()) || (this.position() > requestedCapacity)) { + return; + } + final int bytesToCopy = Math.min(Math.max(requestedCapacity, position()), capacity()) * SIZE_OF_BYTE; + final byte[] newBuffer = byteArrayCache == null ? new byte[requestedCapacity] : byteArrayCache.getArrayExact(requestedCapacity); + copyMemory(buffer, ARRAY_BYTE_BASE_OFFSET, newBuffer, ARRAY_BYTE_BASE_OFFSET, bytesToCopy); + if (byteArrayCache != null) { + byteArrayCache.add(buffer); + } + buffer = newBuffer; + intLimit = newBuffer.length; + } + + /** + * Wraps a given byte array into FastByteBuffer + *

+ * Note it is guaranteed that the type of the array returned by {@link #elements()} will be the same. + * + * @param byteArray an array to wrap. + * @return a new FastByteBuffer of the given size, wrapping the given array. + */ + public static FastByteBuffer wrap(final byte[] byteArray) { + return wrap(byteArray, byteArray.length); + } + + /** + * Wraps a given byte array into FastByteBuffer + *

+ * Note it is guaranteed that the type of the array returned by {@link #elements()} will be the same. + * + * @param byteArray an array to wrap. + * @param length the length of the resulting array list. + * @return a new FastByteBuffer of the given size, wrapping the given array. + */ + public static FastByteBuffer wrap(final byte[] byteArray, final int length) { + return new FastByteBuffer(byteArray, length); + } + + private static void copyMemory(final Object srcBase, final int srcOffset, final Object destBase, final int destOffset, final int nBytes) { + unsafe.copyMemory(srcBase, srcOffset, destBase, destOffset, nBytes); + } + + // Fast UTF-8 byte-array to String(Builder) decode - code originally based on Google's ProtoBuffer implementation and since modified + @SuppressWarnings("PMD") + private static void decodeUTF8(byte[] bytes, int offset, int size, StringBuilder result) { //NOSONAR + // Bitwise OR combines the sign bits so any negative value fails the check. + // N.B. many code snippets are in-lined for performance reasons (~10% performance improvement) ... this is a JIT hot spot. + if ((offset | size | bytes.length - offset - size) < 0) { + throw new ArrayIndexOutOfBoundsException(String.format("buffer length=%d, offset=%d, size=%d", bytes.length, offset, size)); + } + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + result.setLength(size); + + // keep separate int/long counters so we don't have to convert types at every call + int remaining = size; + long readPos = (long) ARRAY_BYTE_BASE_OFFSET + offset; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (remaining > 0) { + final byte byte1 = unsafe.getByte(bytes, readPos); + if (byte1 < 0) { + // is more than one byte ie. non-ASCII (unsigned byte value larger > 127 <-> negative number for signed byte + break; + } + readPos++; + remaining--; + result.setCharAt(resultPos++, (char) byte1); + } + + while (remaining > 0) { + final byte byte1 = unsafe.getByte(bytes, readPos++); + remaining--; + if (byte1 >= 0) { // is one byte (ie. ASCII-type encoding) + result.setCharAt(resultPos++, (char) byte1); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an extra optimized loop to take care of these runs. + while (remaining > 0) { + final byte b = unsafe.getByte(bytes, readPos); + if (b < 0) { // is not one byte + break; + } + readPos++; + remaining--; + result.setCharAt(resultPos++, (char) b); + } + } else if (byte1 < (byte) 0xE0) { // is two bytes + if (remaining < 1) { + throw new IllegalArgumentException(INVALID_UTF_8); + } + final byte byte2 = unsafe.getByte(bytes, readPos++); + remaining--; + final int resultPos1 = resultPos++; + // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and overlong 2-byte, '11000001'. + if (byte1 < (byte) 0xC2) { + throw new IllegalArgumentException(INVALID_UTF_8 + ": Illegal leading byte in 2 bytes UTF"); + } + if (byte2 > (byte) 0xBF) { // is not trailing byte + throw new IllegalArgumentException(INVALID_UTF_8 + ": Illegal trailing byte in 2 bytes UTF"); + } + result.setCharAt(resultPos1, (char) (((byte1 & 0x1F) << 6) | byte2 & 0x3F)); + } else if (byte1 < (byte) 0xF0) { // is three bytes + if (remaining < 2) { + throw new IllegalArgumentException(INVALID_UTF_8); + } + /* byte2 */ + /* byte3 */ + final byte byte2 = unsafe.getByte(bytes, readPos++); + final byte byte3 = unsafe.getByte(bytes, readPos++); + final int resultPos1 = resultPos++; + if (byte2 > (byte) 0xBF // is not trailing byte + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + || byte3 > (byte) 0xBF) { // is not trailing byte + throw new IllegalArgumentException(INVALID_UTF_8); + } + result.setCharAt(resultPos1, (char) (((byte1 & 0x0F) << 12) | ((byte2 & 0x3F) << 6) | byte3 & 0x3F)); + remaining -= 2; + } else { // is four bytes + if (remaining < 3) { + throw new IllegalArgumentException(INVALID_UTF_8); + } + // handle four byte UTF + /* byte2 */ + /* byte3 */ + /* byte4 */ + final byte byte2 = unsafe.getByte(bytes, readPos++); + final byte byte3 = unsafe.getByte(bytes, readPos++); + final byte byte4 = unsafe.getByte(bytes, readPos++); + final int resultPos1 = resultPos++; + if (byte2 > (byte) 0xBF + // Check that 1 <= plane <= 16. Tricky optimized form of: + // valid 4-byte leading byte? + // if (byte1 > (byte) 0xF4 || + // overlong? 4 most significant bits must not all be zero + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // codepoint larger than the highest code point (U+10FFFF)? byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + || byte3 > (byte) 0xBF + || byte4 > (byte) 0xBF) { + throw new IllegalArgumentException(INVALID_UTF_8); + } + final int codepoint = ((byte1 & 0x07) << 18) | ((byte2 & 0x3F) << 12) | ((byte3 & 0x3F) << 6) | byte4 & 0x3F; + result.setCharAt(resultPos1, (char) ((Character.MIN_HIGH_SURROGATE - (Character.MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + (codepoint >>> 10))); + result.setCharAt(resultPos1 + 1, (char) (Character.MIN_LOW_SURROGATE + (codepoint & 0x3ff))); + remaining -= 3; + // 4-byte case requires two chars. + resultPos++; + } + } + + result.setLength(resultPos); + } + + private static int encodeISO8859(final String sequence, final byte[] bytes, final int offset, final int length) { + // encode to ISO_8859_1 + final int base = ARRAY_BYTE_BASE_OFFSET + offset; + for (int i = 0; i < length; i++) { + unsafe.putByte(bytes, (long) base + i, (byte) (sequence.charAt(i) & 0xFF)); + } + return length; + } + + // Fast UTF-8 String (CharSequence) to byte-array encoder - code originally based on Google's ProtoBuffer implementation and since modified + @SuppressWarnings("PMD") + private static int encodeUTF8(final CharSequence sequence, final byte[] bytes, final int offset, final int length) { //NOSONAR + int utf16Length = sequence.length(); + int base = ARRAY_BYTE_BASE_OFFSET + offset; + int i = 0; + int limit = base + length; + // Designed to take advantage of https://wiki.openjdk.java.net/display/HotSpot/RangeCheckElimination + for (char c; i < utf16Length && base + i < limit && (c = sequence.charAt(i)) < 0x80; i++) { + unsafe.putByte(bytes, (long) base + i, (byte) c); + } + if (i == utf16Length) { + return utf16Length; + } + base += i; + for (; i < utf16Length; i++) { + final char c = sequence.charAt(i); + if (c < 0x80 && base < limit) { + unsafe.putByte(bytes, base++, (byte) c); + } else if (c < 0x800 && base <= limit - 2) { // 11 bits, two UTF-8 bytes + unsafe.putByte(bytes, base++, (byte) ((0xF << 6) | (c >>> 6))); + unsafe.putByte(bytes, base++, (byte) (0x80 | (0x3F & c))); + } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && base <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + unsafe.putByte(bytes, base++, (byte) ((0xF << 5) | (c >>> 12))); + unsafe.putByte(bytes, base++, (byte) (0x80 | (0x3F & (c >>> 6)))); + unsafe.putByte(bytes, base++, (byte) (0x80 | (0x3F & c))); + } else if (base <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 bytes + final char low; + if (i + 1 == sequence.length() || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { + throw new IllegalArgumentException("Unpaired surrogate at index " + (i - 1)); + } + final int codePoint = Character.toCodePoint(c, low); + unsafe.putByte(bytes, base++, (byte) ((0xF << 4) | (codePoint >>> 18))); + unsafe.putByte(bytes, base++, (byte) (0x80 | (0x3F & (codePoint >>> 12)))); + unsafe.putByte(bytes, base++, (byte) (0x80 | (0x3F & (codePoint >>> 6)))); + unsafe.putByte(bytes, base++, (byte) (0x80 | (0x3F & codePoint))); + } else { + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + base); + } + } + return base - ARRAY_BYTE_BASE_OFFSET - offset; + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/JsonSerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/JsonSerialiser.java new file mode 100644 index 00000000..ef7ac1ad --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/JsonSerialiser.java @@ -0,0 +1,933 @@ +package io.opencmw.serialiser.spi; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.BiFunction; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoSerialiser; + +import com.jsoniter.JsonIterator; +import com.jsoniter.JsonIteratorPool; +import com.jsoniter.any.Any; +import com.jsoniter.extra.PreciseFloatSupport; +import com.jsoniter.output.EncodingMode; +import com.jsoniter.output.JsonStream; +import com.jsoniter.spi.DecodingMode; +import com.jsoniter.spi.JsonException; + +import de.gsi.dataset.utils.ByteBufferOutputStream; + +@SuppressWarnings({ "PMD.ExcessivePublicCount", "PMD.TooManyMethods" }) // unavoidable: Java does not support templates and primitive types need to be handled one-by-one +public class JsonSerialiser implements IoSerialiser { + public static final String NOT_A_JSON_COMPATIBLE_PROTOCOL = "Not a JSON compatible protocol"; + public static final String JSON_ROOT = "JSON_ROOT"; + private static final int DEFAULT_INITIAL_CAPACITY = 10_000; + private static final int DEFAULT_INDENTATION = 2; + private static final char BRACKET_OPEN = '{'; + private static final char BRACKET_CLOSE = '}'; + public static final char QUOTE = '\"'; + private static final String NULL = "null"; + private static final String ASSIGN = ": "; + private static final String LINE_BREAK = System.getProperty("line.separator"); + public static final String UNCHECKED = "unchecked"; + private final StringBuilder builder = new StringBuilder(DEFAULT_INITIAL_CAPACITY); // NOPMD + private IoBuffer buffer; + private Any root; + private Any tempRoot; + private WireDataFieldDescription parent; + private WireDataFieldDescription lastFieldHeader; + private String queryFieldName; + private boolean hasFieldBefore; + private String indentation = ""; + private BiFunction> fieldSerialiserLookupFunction; + + /** + * @param buffer the backing IoBuffer (see e.g. {@link FastByteBuffer} or{@link ByteBuffer} + */ + public JsonSerialiser(final IoBuffer buffer) { + super(); + this.buffer = buffer; + + // JsonStream.setIndentionStep(DEFAULT_INDENTATION) + // JsonStream.setMode(EncodingMode.REFLECTION_MODE) -- enable as a fall back + // JsonIterator.setMode(DecodingMode.REFLECTION_MODE) -- enable as a fall back + JsonStream.setMode(EncodingMode.DYNAMIC_MODE); + JsonIterator.setMode(DecodingMode.DYNAMIC_MODE_AND_MATCH_FIELD_WITH_HASH); + + try { + PreciseFloatSupport.enable(); + } catch (JsonException e) { + // swallow subsequent enabling exceptions (function is guarded and supposed to be called only once) + } + } + + @Override + public ProtocolInfo checkHeaderInfo() { + // make coarse check (ie. check if first non-null character is a '{' bracket + int count = buffer.position(); + while (buffer.getByte(count) != BRACKET_OPEN && (buffer.getByte(count) == 0 || buffer.getByte(count) == ' ' || buffer.getByte(count) == '\t' || buffer.getByte(count) == '\n')) { + count++; + } + if (buffer.getByte(count) != BRACKET_OPEN) { + throw new IllegalStateException(NOT_A_JSON_COMPATIBLE_PROTOCOL); + } + + try (JsonIterator iter = JsonIteratorPool.borrowJsonIterator()) { + iter.reset(buffer.elements(), 0, buffer.limit()); + tempRoot = root = iter.readAny(); + } catch (IOException e) { + throw new IllegalStateException(NOT_A_JSON_COMPATIBLE_PROTOCOL, e); + } + + final WireDataFieldDescription headerStartField = new WireDataFieldDescription(this, null, JSON_ROOT.hashCode(), JSON_ROOT, DataType.OTHER, buffer.position(), count - 1, -1); + final ProtocolInfo header = new ProtocolInfo(this, headerStartField, JsonSerialiser.class.getCanonicalName(), (byte) 1, (byte) 0, (byte) 0); + parent = lastFieldHeader = headerStartField; + queryFieldName = JSON_ROOT; + return header; + } + + public T deserialiseObject(final T obj) { + try (JsonIterator iter = JsonIterator.parse(buffer.elements(), 0, buffer.limit())) { + return iter.read(obj); + } catch (IOException | JsonException e) { + throw new IllegalStateException(NOT_A_JSON_COMPATIBLE_PROTOCOL, e); + } + } + + @Override + public int[] getArraySizeDescriptor() { + return new int[0]; + } + + @Override + public boolean getBoolean() { + return tempRoot.get(queryFieldName).toBoolean(); + } + + @Override + public boolean[] getBooleanArray(final boolean[] dst, final int length) { + return tempRoot.get(queryFieldName).as(boolean[].class); + } + + @Override + public IoBuffer getBuffer() { + return buffer; + } + + @Override + public byte getByte() { + return (byte) tempRoot.get(queryFieldName).toInt(); + } + + @Override + public byte[] getByteArray(final byte[] dst, final int length) { + return tempRoot.get(queryFieldName).as(byte[].class); + } + + @Override + public char getChar() { + return (char) tempRoot.get(queryFieldName).toInt(); + } + + @Override + public char[] getCharArray(final char[] dst, final int length) { + return tempRoot.get(queryFieldName).as(char[].class); + } + + @Override + @SuppressWarnings(UNCHECKED) + public Collection getCollection(final Collection collection) { + return tempRoot.get(queryFieldName).as(ArrayList.class); + } + + @Override + @SuppressWarnings(UNCHECKED) + public E getCustomData(final FieldSerialiser serialiser) { + return (E) tempRoot.get(queryFieldName); + } + + @Override + public double getDouble() { + return tempRoot.get(queryFieldName).toDouble(); + } + + @Override + public double[] getDoubleArray(final double[] dst, final int length) { + return tempRoot.get(queryFieldName).as(double[].class); + } + + @Override + public > Enum getEnum(final Enum enumeration) { + return null; + } + + @Override + public String getEnumTypeList() { + return null; + } + + @Override + public WireDataFieldDescription getFieldHeader() { + return null; + } + + @Override + public float getFloat() { + return tempRoot.get(queryFieldName).toFloat(); + } + + @Override + public float[] getFloatArray(final float[] dst, final int length) { + return tempRoot.get(queryFieldName).as(float[].class); + } + + @Override + public int getInt() { + return tempRoot.get(queryFieldName).toInt(); + } + + @Override + public int[] getIntArray(final int[] dst, final int length) { + return tempRoot.get(queryFieldName).as(int[].class); + } + + @Override + @SuppressWarnings(UNCHECKED) + public List getList(final List collection) { + return tempRoot.get(queryFieldName).as(List.class); + } + + @Override + public long getLong() { + return tempRoot.get(queryFieldName).toLong(); + } + + @Override + public long[] getLongArray(final long[] dst, final int length) { + return tempRoot.get(queryFieldName).as(long[].class); + } + + @Override + public Map getMap(final Map map) { + return null; + } + + public WireDataFieldDescription getParent() { + return parent; + } + + @Override + @SuppressWarnings(UNCHECKED) + public Queue getQueue(final Queue collection) { + return tempRoot.get(queryFieldName).as(ArrayDeque.class); + } + + @Override + @SuppressWarnings(UNCHECKED) + public Set getSet(final Set collection) { + return tempRoot.get(queryFieldName).as(HashSet.class); + } + + @Override + public short getShort() { + return (short) tempRoot.get(queryFieldName).toLong(); + } + + @Override + public short[] getShortArray(final short[] dst, final int length) { + return tempRoot.get(queryFieldName).as(short[].class); + } + + @Override + public String getString() { + return tempRoot.get(queryFieldName).toString(); + } + + @Override + public String[] getStringArray(final String[] dst, final int length) { + return tempRoot.get(queryFieldName).as(String[].class); + } + + @Override + public String getStringISO8859() { + return tempRoot.get(queryFieldName).toString(); + } + + @Override + public boolean isPutFieldMetaData() { + return false; + } + + @Override + public WireDataFieldDescription parseIoStream(final boolean readHeader) { + try (JsonIterator iter = JsonIteratorPool.borrowJsonIterator()) { + iter.reset(buffer.elements(), 0, buffer.limit()); + tempRoot = root = iter.readAny(); + + final WireDataFieldDescription fieldRoot = new WireDataFieldDescription(this, null, "ROOT".hashCode(), "ROOT", DataType.OTHER, buffer.position(), -1, -1); + parseIoStream(fieldRoot, tempRoot, ""); + + return fieldRoot; + } catch (IOException e) { + throw new IllegalStateException(NOT_A_JSON_COMPATIBLE_PROTOCOL, e); + } + } + + @Override + public void put(final FieldDescription fieldDescription, final Collection collection, final Type valueType) { + put(fieldDescription.getFieldName(), collection, valueType); + } + + @Override + public void put(final FieldDescription fieldDescription, final Enum enumeration) { + put(fieldDescription.getFieldName(), enumeration); + } + + @Override + public void put(final FieldDescription fieldDescription, final Map map, final Type keyType, final Type valueType) { + put(fieldDescription.getFieldName(), map, keyType, valueType); + } + + @Override + public void put(final String fieldName, final Collection collection, final Type valueType) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE).append(ASSIGN).append('['); + if (collection == null || collection.isEmpty()) { + builder.append(']'); + return; + } + final Iterator iter = collection.iterator(); + serialiseObject(iter.next()); + while (iter.hasNext()) { + builder.append(", "); + serialiseObject(iter.next()); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final Enum enumeration) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN + QUOTE).append(enumeration).append(QUOTE); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final Map map, final Type keyType, final Type valueType) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN + '{'); + if (map == null || map.isEmpty()) { + builder.append('}'); + return; + } + final Set> entrySet = map.entrySet(); + boolean isFirst = true; + for (Map.Entry entry : entrySet) { + final V value = entry.getValue(); + if (isFirst) { + isFirst = false; + } else { + builder.append(", "); + } + builder.append(QUOTE).append(entry.getKey()).append(QUOTE + ASSIGN); + + switch (DataType.fromClassType(value.getClass())) { + case CHAR: + builder.append((int) value); + break; + case STRING: + builder.append(QUOTE).append(entry.getValue()).append(QUOTE); + break; + default: + builder.append(value); + break; + } + } + + builder.append('}'); + hasFieldBefore = true; + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final boolean[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final byte[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final char value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final char[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final char[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final double value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final double[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final double[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final float value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final float[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final float[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final int value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final int[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final int[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final long value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final long[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final long[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final short value) { + put(fieldDescription.getFieldName(), value); + } + + @Override + public void put(final FieldDescription fieldDescription, final short[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final short[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final FieldDescription fieldDescription, final String string) { + put(fieldDescription.getFieldName(), string); + } + + @Override + public void put(final FieldDescription fieldDescription, final String[] values, final int n) { + put(fieldDescription.getFieldName(), values, n); + } + + @Override + public void put(final FieldDescription fieldDescription, final String[] values, final int[] dims) { + put(fieldDescription.getFieldName(), values, dims); + } + + @Override + public void put(final String fieldName, final boolean value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final boolean[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN + '['); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final boolean[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final byte value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final byte[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN + '['); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final byte[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final char value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append((int) value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final char[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN + '['); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append((int) values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append((int) values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final char[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final double value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final double[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN + '['); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final double[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final float value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final float[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE).append(ASSIGN).append('['); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final float[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final int value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final int[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append("["); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final int[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final long value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final long[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append("["); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final long[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final short value) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append(value); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final short[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append("["); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(values[0]); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", ").append(values[i]); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final short[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public void put(final String fieldName, final String string) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append("\": \"").append(string).append(QUOTE); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final String[] values, final int n) { + lineBreak(); + builder.append(QUOTE).append(fieldName).append(QUOTE + ASSIGN).append("["); + if (values == null || values.length <= 0) { + builder.append(']'); + return; + } + builder.append(QUOTE).append(values[0]).append(QUOTE); + final int valuesSize = values.length; + final int nElements = n >= 0 ? Math.min(n, valuesSize) : valuesSize; + for (int i = 1; i < nElements; i++) { + builder.append(", \"").append(values[i]).append(QUOTE); + } + builder.append(']'); + hasFieldBefore = true; + } + + @Override + public void put(final String fieldName, final String[] values, final int[] dims) { + put(fieldName, values, getNumberElements(dims)); + } + + @Override + public int putArraySizeDescriptor(final int n) { + return 0; + } + + @Override + public int putArraySizeDescriptor(final int[] dims) { + return 0; + } + + @Override + public WireDataFieldDescription putCustomData(final FieldDescription fieldDescription, final E obj, final Class type, final FieldSerialiser serialiser) { + return null; + } + + @Override + public void putEndMarker(final FieldDescription fieldDescription) { + indentation = indentation.substring(0, Math.max(indentation.length() - DEFAULT_INDENTATION, 0)); + builder.append(LINE_BREAK).append(indentation).append(BRACKET_CLOSE).append(LINE_BREAK); + hasFieldBefore = true; + final byte[] outputStrBytes = builder.toString().getBytes(StandardCharsets.UTF_8); + buffer.ensureAdditionalCapacity(outputStrBytes.length); + System.arraycopy(outputStrBytes, 0, buffer.elements(), buffer.position(), outputStrBytes.length); + buffer.position(buffer.position() + outputStrBytes.length); + builder.setLength(0); + } + + @Override + public WireDataFieldDescription putFieldHeader(final FieldDescription fieldDescription) { + return putFieldHeader(fieldDescription.getFieldName(), fieldDescription.getDataType()); + } + + @Override + public WireDataFieldDescription putFieldHeader(final String fieldName, final DataType dataType) { + lastFieldHeader = new WireDataFieldDescription(this, parent, fieldName.hashCode(), fieldName, dataType, -1, 1, -1); + queryFieldName = fieldName; + return lastFieldHeader; + } + + @Override + public void putHeaderInfo(final FieldDescription... field) { + if (builder.length() > 0) { + final byte[] outputStrBytes = builder.toString().getBytes(StandardCharsets.UTF_8); + buffer.ensureAdditionalCapacity(outputStrBytes.length); + System.arraycopy(outputStrBytes, 0, buffer.elements(), buffer.position(), outputStrBytes.length); + buffer.position(buffer.position() + outputStrBytes.length); + } else { + hasFieldBefore = false; + indentation = ""; + } + builder.setLength(0); + putStartMarker(null); + } + + @Override + public void putStartMarker(final FieldDescription fieldDescription) { + lineBreak(); + if (fieldDescription != null) { + builder.append(QUOTE).append(fieldDescription.getFieldName()).append(QUOTE + ASSIGN); + } + builder.append(BRACKET_OPEN); + indentation = indentation + " ".repeat(DEFAULT_INDENTATION); + builder.append(LINE_BREAK); + builder.append(indentation); + hasFieldBefore = false; + } + + public void serialiseObject(final Object obj) { + if (builder.length() > 0) { + final byte[] outputStrBytes = builder.toString().getBytes(StandardCharsets.UTF_8); + buffer.ensureAdditionalCapacity(outputStrBytes.length); + System.arraycopy(outputStrBytes, 0, buffer.elements(), buffer.position(), outputStrBytes.length); + buffer.position(buffer.position() + outputStrBytes.length); + builder.setLength(0); + } else { + hasFieldBefore = false; + indentation = ""; + } + if (obj == null) { + // serialise null object + builder.append(NULL); + byte[] bytes = builder.toString().getBytes(Charset.defaultCharset()); + System.arraycopy(bytes, 0, buffer.elements(), buffer.position(), bytes.length); + buffer.position(buffer.position() + bytes.length); + builder.setLength(0); + return; + } + try (ByteBufferOutputStream byteOutputStream = new ByteBufferOutputStream(java.nio.ByteBuffer.wrap(buffer.elements()), false)) { + byteOutputStream.position(buffer.position()); + JsonStream.serialize(obj, byteOutputStream); + buffer.position(byteOutputStream.position()); + } catch (IOException e) { + throw new IllegalStateException(NOT_A_JSON_COMPATIBLE_PROTOCOL, e); + } + } + + @Override + public void setBuffer(final IoBuffer buffer) { + this.buffer = buffer; + } + + @Override + public void setPutFieldMetaData(final boolean putFieldMetaData) { + // json does not support metadata + } + + @Override + public void setQueryFieldName(final String fieldName, final int dataStartPosition) { + if (fieldName == null || fieldName.isBlank()) { + throw new IllegalArgumentException("fieldName must not be null or blank: " + fieldName); + } + if (root == null) { + throw new IllegalArgumentException("JSON Any root hasn't been analysed/parsed yet"); + } + this.queryFieldName = fieldName; + // buffer.position(dataStartPosition); // N.B. not needed at this time + } + + @Override + public void updateDataEndMarker(final WireDataFieldDescription fieldHeader) { + // not needed + } + + private int getNumberElements(final int[] dims) { + int n = 1; + for (final int dim : dims) { + n *= dim; + } + return n; + } + + private void lineBreak() { + if (hasFieldBefore) { + builder.append(','); + builder.append(LINE_BREAK); + builder.append(indentation); + } + } + + private void parseIoStream(final WireDataFieldDescription fieldRoot, final Any any, final String fieldName) { + if (!(any.object() instanceof Map) || any.size() == 0) { + return; + } + + final Map map = any.asMap(); + final WireDataFieldDescription putStartMarker = new WireDataFieldDescription(this, fieldRoot, fieldName.hashCode(), fieldName, DataType.START_MARKER, 0, -1, -1); + for (Map.Entry child : map.entrySet()) { + final String childName = child.getKey(); + final Any childAny = map.get(childName); + final Object data = childAny.object(); + if (data instanceof Map) { + parseIoStream(putStartMarker, childAny, childName); + } else if (data != null) { + new WireDataFieldDescription(this, putStartMarker, childName.hashCode(), childName, DataType.fromClassType(data.getClass()), 0, -1, -1); // NOPMD - necessary to allocate inside loop + } + } + // add if necessary: + // new WireDataFieldDescription(this, fieldRoot, fieldName.hashCode(), fieldName, DataType.END_MARKER, 0, -1, -1) + } + + @Override + public void setFieldSerialiserLookupFunction(final BiFunction> serialiserLookupFunction) { + this.fieldSerialiserLookupFunction = serialiserLookupFunction; + } + + @Override + public BiFunction> getSerialiserLookupFunction() { + return fieldSerialiserLookupFunction; + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/ProtocolInfo.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/ProtocolInfo.java new file mode 100644 index 00000000..ef2515f1 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/ProtocolInfo.java @@ -0,0 +1,65 @@ +package io.opencmw.serialiser.spi; + +import io.opencmw.serialiser.IoSerialiser; + +public class ProtocolInfo extends WireDataFieldDescription { + private final WireDataFieldDescription fieldHeader; + private final String producerName; + private final byte versionMajor; + private final byte versionMinor; + private final byte versionMicro; + + public ProtocolInfo(final IoSerialiser source, final WireDataFieldDescription fieldDescription, final String producer, final byte major, final byte minor, final byte micro) { + super(source, null, fieldDescription.hashCode(), fieldDescription.getFieldName(), fieldDescription.getDataType(), fieldDescription.getFieldStart(), fieldDescription.getDataStartOffset(), fieldDescription.getDataSize()); + this.fieldHeader = fieldDescription; + producerName = producer; + versionMajor = major; + versionMinor = minor; + versionMicro = micro; + } + + @Override + public boolean equals(final Object obj) { + if (!(obj instanceof ProtocolInfo)) { + return false; + } + final ProtocolInfo other = (ProtocolInfo) obj; + return other.isCompatible(); + } + + public WireDataFieldDescription getFieldHeader() { + return fieldHeader; + } + + public String getProducerName() { + return producerName; + } + + public byte getVersionMajor() { + return versionMajor; + } + + public byte getVersionMicro() { + return versionMicro; + } + + public byte getVersionMinor() { + return versionMinor; + } + + @Override + public int hashCode() { + return producerName.hashCode(); + } + + public boolean isCompatible() { + // N.B. no API changes within the same 'major.minor'- version + // micro.version tracks possible benin additions & internal bug-fixes + return getVersionMajor() <= BinarySerialiser.VERSION_MAJOR && getVersionMinor() <= BinarySerialiser.VERSION_MINOR; + } + + @Override + public String toString() { + return super.toString() + String.format(" serialiser: %s-v%d.%d.%d", getProducerName(), getVersionMajor(), getVersionMinor(), getVersionMicro()); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/WireDataFieldDescription.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/WireDataFieldDescription.java new file mode 100644 index 00000000..7cc8ee9f --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/WireDataFieldDescription.java @@ -0,0 +1,314 @@ +package io.opencmw.serialiser.spi; + +import java.nio.CharBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.utils.ClassUtils; + +/** + * Field header descriptor + * + * @author rstein + */ +public class WireDataFieldDescription implements FieldDescription { + private static final Logger LOGGER = LoggerFactory.getLogger(WireDataFieldDescription.class); + private final String fieldName; + private final int fieldNameHashCode; + private final DataType dataType; + private final List children = new ArrayList<>(); + private final FieldDescription parent; + private final int fieldStart; + private final int fieldDataStart; + private final int dataStartOffset; + // local references to source buffer needed for parsing + private final IoSerialiser ioSerialiser; + private String fieldUnit; + private String fieldDescription; + private String fieldDirection; + private List fieldGroups; + private int dataSize; + + /** + * Constructs new serializer field header + * + * @param source the referenced IoBuffer (if any) + * @param parent the optional parent field header (for cascaded objects) + * @param fieldNameHashCode the fairly-unique hash-code of the field name, + * N.B. checked during 1st iteration against fieldName, if no collisions are present then + * this check is being suppressed + * @param fieldName the clear text field name description + * @param dataType the data type of that field + * @param fieldStart the absolute buffer position from which the field header can be parsed + * @param dataStartOffset the position from which the actual data can be parsed onwards + * @param dataSize the expected number of bytes to skip the data block + */ + public WireDataFieldDescription(final IoSerialiser source, final FieldDescription parent, final int fieldNameHashCode, final String fieldName, final DataType dataType, // + final int fieldStart, final int dataStartOffset, final int dataSize) { + ioSerialiser = source; + this.parent = parent; + this.fieldNameHashCode = fieldNameHashCode; + this.fieldName = fieldName; + this.dataType = dataType; + this.fieldStart = fieldStart; + this.dataStartOffset = dataStartOffset; + this.dataSize = dataSize; + this.fieldDataStart = fieldStart + dataStartOffset; + + if (this.parent != null /*&& !this.parent.getChildren().contains(this)*/) { + this.parent.getChildren().add(this); + } + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof FieldDescription)) { + return false; + } + FieldDescription other = (FieldDescription) obj; + if (this.getFieldNameHashCode() != other.getFieldNameHashCode()) { + return false; + } + + if (this.getDataType() != other.getDataType()) { + return false; + } + + return this.getFieldName().equals(other.getFieldName()); + } + + @Override + public FieldDescription findChildField(final String fieldName) { + return findChildField(fieldName.hashCode(), fieldName); + } + + @Override + public FieldDescription findChildField(final int fieldNameHashCode, final String fieldName) { + for (final FieldDescription field : children) { //NOSONAR + final String name = field.getFieldName(); + if (name == fieldName) { // NOSONAR NOPMD early return if the same String object reference + return field; + } + if (field.hashCode() == fieldNameHashCode && name.equals(fieldName)) { + return field; + } + } + return null; + } + + @Override + public List getChildren() { + return children; + } + + @Override + public int getDataSize() { + return dataSize; + } + + public void setDataSize(final int size) { + dataSize = size; + } + + @Override + public int getDataStartOffset() { + return dataStartOffset; + } + + @Override + public int getDataStartPosition() { + return fieldDataStart; + } + + @Override + public DataType getDataType() { + return dataType; + } + + @Override + public String getFieldDescription() { + return fieldDescription; + } + + public void setFieldDescription(final String fieldDescription) { + this.fieldDescription = fieldDescription; + } + + @Override + public String getFieldDirection() { + return fieldDirection; + } + + public void setFieldDirection(final String fieldDirection) { + this.fieldDirection = fieldDirection; + } + + @Override + public List getFieldGroups() { + return fieldGroups; + } + + public void setFieldGroups(final List fieldGroups) { + this.fieldGroups = fieldGroups; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public int getFieldNameHashCode() { + return fieldNameHashCode; + } + + @Override + public int getFieldStart() { + return fieldStart; + } + + @Override + public String getFieldUnit() { + return fieldUnit; + } + + public void setFieldUnit(final String fieldUnit) { + this.fieldUnit = fieldUnit; + } + + /** + * @return raw ioSerialiser reference this field was retrieved with the position in the underlying IoBuffer at the to be read field + * N.B. this is a safe convenience method and not performance optimised + * @param overwriteType optional DataType as which the data should be interpreted + */ + public Object data(DataType... overwriteType) { + ioSerialiser.setQueryFieldName(fieldName, fieldDataStart); + switch (overwriteType.length == 0 ? this.dataType : overwriteType[0]) { + case START_MARKER: + case END_MARKER: + return null; + case BOOL: + return ioSerialiser.getBoolean(); + case BYTE: + return ioSerialiser.getByte(); + case SHORT: + return ioSerialiser.getShort(); + case INT: + return ioSerialiser.getInt(); + case LONG: + return ioSerialiser.getLong(); + case FLOAT: + return ioSerialiser.getFloat(); + case DOUBLE: + return ioSerialiser.getDouble(); + case CHAR: + return ioSerialiser.getChar(); + case STRING: + return ioSerialiser.getString(); + case BOOL_ARRAY: + return ioSerialiser.getBooleanArray(); + case BYTE_ARRAY: + return ioSerialiser.getByteArray(); + case SHORT_ARRAY: + return ioSerialiser.getShortArray(); + case INT_ARRAY: + return ioSerialiser.getIntArray(); + case LONG_ARRAY: + return ioSerialiser.getLongArray(); + case FLOAT_ARRAY: + return ioSerialiser.getFloatArray(); + case DOUBLE_ARRAY: + return ioSerialiser.getDoubleArray(); + case CHAR_ARRAY: + return ioSerialiser.getCharArray(); + case STRING_ARRAY: + return ioSerialiser.getStringArray(); + case ENUM: + return ioSerialiser.getEnum(null); + case LIST: + return ioSerialiser.getList(null); + case MAP: + return ioSerialiser.getMap(null); + case QUEUE: + return ioSerialiser.getQueue(null); + case SET: + return ioSerialiser.getSet(null); + case COLLECTION: + return ioSerialiser.getCollection(null); + case OTHER: + return ioSerialiser.getCustomData(null); + default: + throw new IllegalStateException("unknown dataType = " + dataType); + } + } + + /** + * @return raw ioSerialiser reference this field was retrieved from w/o changing the position in the underlying IoBuffer + */ + public IoSerialiser getIoSerialiser() { + return ioSerialiser; + } + + @Override + public FieldDescription getParent() { + return parent; + } + + @Override + public Class getType() { + return dataType.getClassTypes().get(0); + } + + @Override + public int hashCode() { + return fieldNameHashCode; + } + + @Override + public boolean isAnnotationPresent() { + return fieldUnit != null || fieldDescription != null || fieldDirection != null || (fieldGroups != null && !fieldGroups.isEmpty()); + } + + @Override + public void printFieldStructure() { + if (parent == null) { + LOGGER.atInfo().log("FielHeader structure (no parent):"); + } else { + LOGGER.atInfo().addArgument(parent).log("FielHeader structure (parent: {}):"); + printFieldStructure(this, 0); + } + printFieldStructure(this, 0); + } + + @Override + public String toString() { + return String.format("[fieldName=%s, fieldType=%s]", fieldName, dataType.getAsString()); + } + + protected static void printFieldStructure(final FieldDescription field, final int recursionLevel) { + final String mspace = spaces((recursionLevel) *ClassUtils.getIndentationNumberOfSpace()); + LOGGER.atInfo().addArgument(mspace).addArgument(field.toString()).log("{}{}"); + if (field.isAnnotationPresent()) { + LOGGER.atInfo().addArgument(mspace) // + .addArgument(field.getFieldUnit()) + .addArgument(field.getFieldDescription()) + .addArgument(field.getFieldDirection()) + .addArgument(field.getFieldGroups()) + .log("{} "); + } + field.getChildren().forEach(f -> printFieldStructure(f, recursionLevel + 1)); + } + + private static String spaces(final int spaces) { + return CharBuffer.allocate(spaces).toString().replace('\0', ' '); + } +} \ No newline at end of file diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/DataSetSerialiser.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/DataSetSerialiser.java new file mode 100644 index 00000000..0768e371 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/DataSetSerialiser.java @@ -0,0 +1,518 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.InputMismatchException; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.WireDataFieldDescription; +import io.opencmw.serialiser.utils.AssertUtils; +import io.opencmw.serialiser.utils.GenericsHelper; + +import de.gsi.dataset.AxisDescription; +import de.gsi.dataset.DataSet; +import de.gsi.dataset.DataSetError; +import de.gsi.dataset.DataSetMetaData; +import de.gsi.dataset.GridDataSet; +import de.gsi.dataset.spi.AbstractDataSet; +import de.gsi.dataset.spi.DataSetBuilder; +import de.gsi.dataset.spi.utils.MathUtils; +import de.gsi.dataset.spi.utils.StringHashMapList; + +/** + * Class to efficiently serialise and de-serialise DataSet objects into binary byte arrays. The performance can be tuned + * through: + *

    + *
  • using floats (ie. memory-IO vs network-IO bound serialisation), or
    • + *
  • via {@link #setDataLablesSerialised(boolean)} (default: true) to control whether data labels and styles shall be processed
    • + *
  • via {@link #setMetaDataSerialised(boolean)} (default: true) to control whether meta data shall be processed
    • + *
+ * + * @author rstein + */ +public class DataSetSerialiser { // NOPMD + private static final Logger LOGGER = LoggerFactory.getLogger(DataSetSerialiser.class); + private static final String DATA_SET_NAME = "dataSetName"; + private static final String DIMENSIONS = "nDims"; + private static final String ARRAY_PREFIX = "array"; + private static final String EN_PREFIX = "en"; + private static final String EP_PREFIX = "ep"; + private static final String AXIS = "axis"; + private static final String NAME = "name"; + private static final String UNIT = "unit"; + private static final String MIN = "Min"; + private static final String MAX = "Max"; + private static final String META_INFO = "metaInfo"; + private static final String ERROR_LIST = "errorList"; + private static final String WARNING_LIST = "warningList"; + private static final String INFO_LIST = "infoList"; + private static final String DATA_STYLES = "dataStyles"; + private static final String DATA_LABELS = "dataLabels"; + private final IoSerialiser ioSerialiser; + private boolean transmitDataLabels = true; + private boolean transmitMetaData = true; + + private DataSetSerialiser(final IoSerialiser ioSerialiser) { + this.ioSerialiser = ioSerialiser; + } + + public boolean isDataLablesSerialised() { + return transmitDataLabels; + } + + public boolean isMetaDataSerialised() { + return transmitMetaData; + } + + /** + * Read a Dataset from a byte array containing comma separated values.
+ * The data format is a custom extension of csv with an additional #-commented Metadata Header and a $-commented + * column header. Expects the following columns in this order to be present: index, x, y, eyn, eyp. + * + * @return DataSet with the data and metadata read from the file + */ + public DataSet read() { // NOPMD + return read(null); + } + + /** + * Read a Dataset from a byte array containing comma separated values.
+ * The data format is a custom extension of csv with an additional #-commented Metadata Header and a $-commented + * column header. Expects the following columns in this order to be present: index, x, y, eyn, eyp. + * + * @param dataSet inplace DataSet that is being overwritten if non-null and {@link DataSet#set(DataSet, boolean)} is implemented + * @return DataSet with the data and metadata read from the file + */ + public DataSet read(final DataSet dataSet) { // NOPMD + final DataSetBuilder builder = new DataSetBuilder(); + FieldDescription root = ioSerialiser.parseIoStream(false); + final FieldDescription fieldRoot = root.getChildren().get(0); + // parsed until end of buffer + + parseHeaders(ioSerialiser, builder, fieldRoot); + + if (isMetaDataSerialised()) { + parseMetaData(ioSerialiser, builder, fieldRoot); + } + + if (isDataLablesSerialised()) { + parseDataLabels(builder, fieldRoot); + } + + parseNumericData(ioSerialiser, builder, dataSet, fieldRoot); + + if (root.getChildren().size() != 2) { + throw new IllegalArgumentException("fieldRoot children-count != 2: " + fieldRoot.getChildren().size()); + } + final FieldDescription endMarker = root.getChildren().get(1); + if (endMarker.getDataType() != DataType.END_MARKER) { + throw new IllegalArgumentException("fieldRoot END_MARKER expected but found: " + endMarker); + } + // move read position to after end marker + ioSerialiser.getBuffer().position(endMarker.getDataStartPosition()); + ioSerialiser.updateDataEndMarker((WireDataFieldDescription) endMarker); + if (dataSet == null) { + return builder.build(); + } + + // in-place update preserving existing listener, N.B: 'false' is important <-> inplace copy + return dataSet.set(builder.build(), false); + } + + public DataSetSerialiser setDataLablesSerialised(final boolean state) { + transmitDataLabels = state; + return this; + } + + public DataSetSerialiser setMetaDataSerialised(final boolean state) { + transmitMetaData = state; + return this; + } + + /** + * Write data set into byte buffer. + * + * @param dataSet The DataSet to export + * @param asFloat {@code true}: encode data as binary floats (smaller size, performance), or {@code false} as double + * (better precision) + */ + public void write(final DataSet dataSet, final boolean asFloat) { + AssertUtils.notNull("dataSet", dataSet); + AssertUtils.notNull("ioSerialiser", ioSerialiser); + final String dataStartMarkerName = "START_MARKER_DATASET:" + dataSet.getName(); + final WireDataFieldDescription dataStartMarker = new WireDataFieldDescription(ioSerialiser, null, dataStartMarkerName.hashCode(), dataStartMarkerName, DataType.OTHER, -1, -1, -1); + ioSerialiser.putStartMarker(dataStartMarker); + + writeHeaderDataToStream(dataSet); + + if (isMetaDataSerialised()) { + writeMetaDataToStream(dataSet); + } + + if (isDataLablesSerialised()) { + writeDataLabelsToStream(dataSet); + } + + if (asFloat) { + writeNumericBinaryDataToBufferFloat(dataSet); + + } else { + writeNumericBinaryDataToBufferDouble(dataSet); + } + + final String dataEndMarkerName = "END_MARKER_DATASET:" + dataSet.getName(); + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); + } + + protected FieldDescription checkFieldCompatibility(final FieldDescription rootField, final int fieldNameHashCode, final String fieldName, final DataType... requireDataTypes) { + FieldDescription fieldHeader = rootField.findChildField(fieldNameHashCode, fieldName); + if (fieldHeader == null) { + return null; + } + + boolean foundMatchingDataType = false; + for (DataType dataType : requireDataTypes) { + if (fieldHeader.getDataType().equals(dataType)) { + foundMatchingDataType = true; + break; + } + } + if (!foundMatchingDataType) { + throw new InputMismatchException(fieldName + " is type " + fieldHeader.getDataType() + " vs. required type " + Arrays.asList(requireDataTypes).toString()); + } + + ioSerialiser.getBuffer().position(fieldHeader.getDataStartPosition()); + return fieldHeader; + } + + protected static int getDimIndex(String fieldName, String prefix) { + try { + return Integer.parseInt(fieldName.substring(prefix.length())); + } catch (NumberFormatException | IndexOutOfBoundsException e) { + LOGGER.atWarn().addArgument(fieldName).log("Invalid field name: {}"); + return -1; + } + } + + protected static double[] getDoubleArray(final IoSerialiser ioSerialiser, final double[] origArray, final DataType dataType) { + switch (dataType) { + case BOOL_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getBooleanArray()); + case BYTE_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getByteArray()); + case SHORT_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getShortArray()); + case INT_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getIntArray()); + case LONG_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getLongArray()); + case FLOAT_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getFloatArray()); + case DOUBLE_ARRAY: + return ioSerialiser.getDoubleArray(origArray); + case CHAR_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getCharArray()); + case STRING_ARRAY: + return GenericsHelper.toDoublePrimitive(ioSerialiser.getStringArray()); + default: + throw new IllegalArgumentException("dataType '" + dataType + "' is not an array"); + } + } + + protected void parseDataLabels(final DataSetBuilder builder, final FieldDescription fieldRoot) { + if (checkFieldCompatibility(fieldRoot, DATA_LABELS.hashCode(), DATA_LABELS, DataType.MAP) != null) { + Map map = new HashMap<>(); // NOPMD - thread-safe usage + map = ioSerialiser.getMap(map); + builder.setDataLabelMap(map); + } + + if (checkFieldCompatibility(fieldRoot, DATA_STYLES.hashCode(), DATA_STYLES, DataType.MAP) != null) { + Map map = new HashMap<>(); // NOPMD - thread-safe usage + map = ioSerialiser.getMap(map); + builder.setDataStyleMap(map); + } + } + + protected void parseHeaders(final IoSerialiser ioSerialiser, final DataSetBuilder builder, final FieldDescription fieldRoot) { + // read strings + if (checkFieldCompatibility(fieldRoot, DATA_SET_NAME.hashCode(), DATA_SET_NAME, DataType.STRING) != null) { + builder.setName(ioSerialiser.getBuffer().getString()); + } + + if (checkFieldCompatibility(fieldRoot, DIMENSIONS.hashCode(), DIMENSIONS, DataType.INT) != null) { + builder.setDimension(ioSerialiser.getBuffer().getInt()); + } + + // check for axis descriptions (all fields starting with AXIS) + for (FieldDescription fieldDescription : fieldRoot.getChildren()) { + parseHeader(ioSerialiser, builder, fieldDescription); + } + } + + protected void parseMetaData(final IoSerialiser ioSerialiser, final DataSetBuilder builder, final FieldDescription rootField) { + if (checkFieldCompatibility(rootField, INFO_LIST.hashCode(), INFO_LIST, DataType.STRING_ARRAY) != null) { + builder.setMetaInfoList(ioSerialiser.getStringArray()); + } + + if (checkFieldCompatibility(rootField, WARNING_LIST.hashCode(), WARNING_LIST, DataType.STRING_ARRAY) != null) { + builder.setMetaWarningList(ioSerialiser.getStringArray()); + } + + if (checkFieldCompatibility(rootField, ERROR_LIST.hashCode(), ERROR_LIST, DataType.STRING_ARRAY) != null) { + builder.setMetaErrorList(ioSerialiser.getStringArray()); + } + + if (checkFieldCompatibility(rootField, META_INFO.hashCode(), META_INFO, DataType.MAP) != null) { + Map map = new HashMap<>(); // NOPMD - thread-safe usage + map = ioSerialiser.getMap(map); + builder.setMetaInfoMap(map); + } + } + + protected void parseNumericData(final IoSerialiser ioSerialiser, final DataSetBuilder builder, final DataSet origDataSet, final FieldDescription rootField) { + // check for numeric data + for (FieldDescription fieldDescription : rootField.getChildren()) { + final String fieldName = fieldDescription.getFieldName(); + if (fieldName == null || (fieldDescription.getDataType() != DataType.DOUBLE_ARRAY && fieldDescription.getDataType() != DataType.FLOAT_ARRAY)) { + continue; + } + if (fieldName.startsWith(ARRAY_PREFIX)) { + readValues(ioSerialiser, builder, origDataSet, fieldDescription, fieldName); + } else if (fieldName.startsWith(EP_PREFIX)) { + readPosError(ioSerialiser, builder, origDataSet, fieldDescription, fieldName); + } else if (fieldName.startsWith(EN_PREFIX)) { + readNegError(ioSerialiser, builder, origDataSet, fieldDescription, fieldName); + } + } + } + + @SuppressWarnings("PMD.NPathComplexity") + protected void writeDataLabelsToStream(final DataSet dataSet) { + if (dataSet instanceof AbstractDataSet) { + final StringHashMapList labelMap = ((AbstractDataSet) dataSet).getDataLabelMap(); + if (!labelMap.isEmpty()) { + ioSerialiser.put(DATA_LABELS, labelMap, Integer.class, String.class); + } + final StringHashMapList styleMap = ((AbstractDataSet) dataSet).getDataStyleMap(); + if (!styleMap.isEmpty()) { + ioSerialiser.put(DATA_STYLES, styleMap, Integer.class, String.class); + } + return; + } + + final int dataCount = dataSet.getDataCount(); + final Map labelMap = new HashMap<>(); // NOPMD - protected by lock and faster + for (int index = 0; index < dataCount; index++) { + final String label = dataSet.getDataLabel(index); + if ((label != null) && !label.isEmpty()) { + labelMap.put(index, label); + } + } + if (!labelMap.isEmpty()) { + ioSerialiser.put(DATA_LABELS, labelMap, Integer.class, String.class); + } + + final Map styleMap = new HashMap<>(); // NOPMD - protected by lock and faster + for (int index = 0; index < dataCount; index++) { + final String style = dataSet.getStyle(index); + if ((style != null) && !style.isEmpty()) { + styleMap.put(index, style); + } + } + if (!styleMap.isEmpty()) { + ioSerialiser.put(DATA_STYLES, styleMap, Integer.class, String.class); + } + } + + protected void writeHeaderDataToStream(final DataSet dataSet) { + // common header data + ioSerialiser.put(DATA_SET_NAME, dataSet.getName()); + ioSerialiser.put(DIMENSIONS, dataSet.getDimension()); + final List axisDescriptions = dataSet.getAxisDescriptions(); + StringBuilder builder = new StringBuilder(60); + for (int i = 0; i < axisDescriptions.size(); i++) { + builder.setLength(0); + final String prefix = builder.append(AXIS).append(i).append('.').toString(); + builder.setLength(0); + final String name = builder.append(prefix).append(NAME).toString(); + builder.setLength(0); + final String unit = builder.append(prefix).append(UNIT).toString(); + builder.setLength(0); + final String minName = builder.append(prefix).append(MIN).toString(); + builder.setLength(0); + final String maxName = builder.append(prefix).append(MAX).toString(); + + ioSerialiser.put(name, dataSet.getAxisDescription(i).getName()); + ioSerialiser.put(unit, dataSet.getAxisDescription(i).getUnit()); + ioSerialiser.put(minName, dataSet.getAxisDescription(i).getMin()); + ioSerialiser.put(maxName, dataSet.getAxisDescription(i).getMax()); + } + } + + protected void writeMetaDataToStream(final DataSet dataSet) { + if (!(dataSet instanceof DataSetMetaData)) { + return; + } + final DataSetMetaData metaDataSet = (DataSetMetaData) dataSet; + + ioSerialiser.put(INFO_LIST, metaDataSet.getInfoList().toArray(new String[0])); + ioSerialiser.put(WARNING_LIST, metaDataSet.getWarningList().toArray(new String[0])); + ioSerialiser.put(ERROR_LIST, metaDataSet.getErrorList().toArray(new String[0])); + ioSerialiser.put(META_INFO, metaDataSet.getMetaInfo(), String.class, String.class); + } + + /** + * @param dataSet to be exported + */ + protected void writeNumericBinaryDataToBufferDouble(final DataSet dataSet) { + final int nDim = dataSet.getDimension(); + if (dataSet instanceof GridDataSet) { + GridDataSet gridDataSet = (GridDataSet) dataSet; + for (int dimIndex = 0; dimIndex < nDim; dimIndex++) { + final boolean gridDimension = dimIndex < gridDataSet.getNGrid(); + final int nsamples = gridDimension ? gridDataSet.getShape(dimIndex) : dataSet.getDataCount(); + final double[] values = gridDimension ? gridDataSet.getGridValues(dimIndex) : dataSet.getValues(dimIndex); + ioSerialiser.put(ARRAY_PREFIX + dimIndex, values, nsamples); + } + return; // GridDataSet does not provide errors + } + for (int dimIndex = 0; dimIndex < nDim; dimIndex++) { + final int nsamples = dataSet.getDataCount(); + ioSerialiser.put(ARRAY_PREFIX + dimIndex, dataSet.getValues(dimIndex), nsamples); + } + if (!(dataSet instanceof DataSetError)) { + return; // data set does not have any error definition + } + final DataSetError ds = (DataSetError) dataSet; + for (int dimIndex = 0; dimIndex < nDim; dimIndex++) { + final int nsamples = dataSet.getDataCount(); + switch (ds.getErrorType(dimIndex)) { + case SYMMETRIC: + ioSerialiser.put(EP_PREFIX + dimIndex, ds.getErrorsPositive(dimIndex), nsamples); + break; + case ASYMMETRIC: + ioSerialiser.put(EN_PREFIX + dimIndex, ds.getErrorsNegative(dimIndex), nsamples); + ioSerialiser.put(EP_PREFIX + dimIndex, ds.getErrorsPositive(dimIndex), nsamples); + break; + case NO_ERROR: + default: + break; + } + } + } + + /** + * @param dataSet to be exported + */ + protected void writeNumericBinaryDataToBufferFloat(final DataSet dataSet) { + final int nDim = dataSet.getDimension(); + if (dataSet instanceof GridDataSet) { + GridDataSet gridDataSet = (GridDataSet) dataSet; + for (int dimIndex = 0; dimIndex < nDim; dimIndex++) { + final boolean gridDimension = dimIndex < gridDataSet.getNGrid(); + final int nsamples = gridDimension ? gridDataSet.getShape(dimIndex) : dataSet.getDataCount(); + final float[] values = MathUtils.toFloats(gridDimension ? gridDataSet.getGridValues(dimIndex) : dataSet.getValues(dimIndex)); + ioSerialiser.put(ARRAY_PREFIX + dimIndex, values, nsamples); + } + return; // GridDataSet does not provide errors + } + for (int dimIndex = 0; dimIndex < nDim; dimIndex++) { + final int nsamples = dataSet.getDataCount(); + ioSerialiser.put(ARRAY_PREFIX + dimIndex, MathUtils.toFloats(dataSet.getValues(dimIndex)), nsamples); + } + + if (!(dataSet instanceof DataSetError)) { + return; // data set does not have any error definition + } + + final DataSetError ds = (DataSetError) dataSet; + for (int dimIndex = 0; dimIndex < nDim; dimIndex++) { + final int nsamples = dataSet.getDataCount(); + switch (ds.getErrorType(dimIndex)) { + case SYMMETRIC: + ioSerialiser.put(EP_PREFIX + dimIndex, MathUtils.toFloats(ds.getErrorsPositive(dimIndex)), nsamples); + break; + case ASYMMETRIC: + ioSerialiser.put(EN_PREFIX + dimIndex, MathUtils.toFloats(ds.getErrorsNegative(dimIndex)), nsamples); + ioSerialiser.put(EP_PREFIX + dimIndex, MathUtils.toFloats(ds.getErrorsPositive(dimIndex)), nsamples); + break; + case NO_ERROR: + default: + break; + } + } + } + + private void parseHeader(final IoSerialiser ioSerialiser, final DataSetBuilder builder, FieldDescription fieldDescription) { + final String fieldName = fieldDescription.getFieldName(); + if (fieldName == null || !fieldName.startsWith(AXIS)) { + return; // not axis related field + } + final String[] parsed = fieldName.split("\\."); + if (parsed.length <= 1) { + return; // couldn't parse axis field + } + final int dimension = getDimIndex(parsed[0], AXIS); + if (dimension < 0) { + return; // couldn't parse dimIndex + } + ioSerialiser.getBuffer().position(fieldDescription.getDataStartPosition()); + switch (parsed[1]) { + case MIN: + builder.setAxisMin(dimension, ioSerialiser.getBuffer().getDouble()); + break; + case MAX: + builder.setAxisMax(dimension, ioSerialiser.getBuffer().getDouble()); + break; + case NAME: + builder.setAxisName(dimension, ioSerialiser.getBuffer().getString()); + break; + case UNIT: + builder.setAxisUnit(dimension, ioSerialiser.getBuffer().getString()); + break; + default: + LOGGER.atWarn().addArgument(parsed[1]).log("parseHeader(): encountered unknown tag {} - ignore"); + break; + } + } + + private void readNegError(final IoSerialiser ioSerialiser, final DataSetBuilder builder, final DataSet origDataSet, FieldDescription fieldDescription, final String fieldName) { + int dimIndex = getDimIndex(fieldName, EN_PREFIX); + if (dimIndex >= 0) { + ioSerialiser.getBuffer().position(fieldDescription.getDataStartPosition()); + final double[] origErrorArray = (origDataSet instanceof DataSetError) ? ((DataSetError) origDataSet).getErrorsNegative(dimIndex) : null; + builder.setNegErrorNoCopy(dimIndex, getDoubleArray(ioSerialiser, origErrorArray, fieldDescription.getDataType())); + } + } + + private void readPosError(final IoSerialiser ioSerialiser, final DataSetBuilder builder, final DataSet origDataSet, FieldDescription fieldDescription, + final String fieldName) { + int dimIndex = getDimIndex(fieldName, EP_PREFIX); + if (dimIndex >= 0) { + ioSerialiser.getBuffer().position(fieldDescription.getDataStartPosition()); + final double[] origErrorArray = (origDataSet instanceof DataSetError) ? ((DataSetError) origDataSet).getErrorsPositive(dimIndex) : null; + builder.setPosErrorNoCopy(dimIndex, getDoubleArray(ioSerialiser, origErrorArray, fieldDescription.getDataType())); + } + } + + private void readValues(final IoSerialiser ioSerialiser, final DataSetBuilder builder, final DataSet origDataSet, FieldDescription fieldDescription, + final String fieldName) { + int dimIndex = getDimIndex(fieldName, ARRAY_PREFIX); + if (dimIndex >= 0) { + ioSerialiser.getBuffer().position(fieldDescription.getDataStartPosition()); + builder.setValuesNoCopy(dimIndex, getDoubleArray(ioSerialiser, origDataSet == null ? null : origDataSet.getValues(dimIndex), fieldDescription.getDataType())); + } + } + + public static DataSetSerialiser withIoSerialiser(final IoSerialiser ioSerialiser) { + return new DataSetSerialiser(ioSerialiser); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldBoxedValueArrayHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldBoxedValueArrayHelper.java new file mode 100644 index 00000000..c7664724 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldBoxedValueArrayHelper.java @@ -0,0 +1,75 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; + +import de.gsi.dataset.utils.GenericsHelper; + +/** + * helper class to register default serialiser for boxed array primitive types (ie. Boolean[], Byte[], Short[], ..., + * double[]) w/o String[] (already part of the {@link FieldPrimitiveValueHelper} + * + * @author rstein + */ +public final class FieldBoxedValueArrayHelper { + public static final String NOT_SUPPORTED_FOR_PRIMITIVES = "return function not supported for primitive types"; + + private FieldBoxedValueArrayHelper() { + // utility class + } + + /** + * registers default serialiser for boxed array primitive types (ie. Boolean[], Byte[], Short[], ..., Double[]) + * + * @param serialiser for which the field serialisers should be registered + */ + public static void register(final IoClassSerialiser serialiser) { + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getBooleanArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toBoolPrimitive((Boolean[]) field.getField().get(obj))), // writer + Boolean[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getByteArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toBytePrimitive((Byte[]) field.getField().get(obj))), // writer + Byte[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getCharArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toCharPrimitive((Character[]) field.getField().get(obj))), // writer + Character[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getShortArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toShortPrimitive((Short[]) field.getField().get(obj))), // writer + Short[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getIntArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toIntegerPrimitive((Integer[]) field.getField().get(obj))), // writer + Integer[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getLongArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toLongPrimitive((Long[]) field.getField().get(obj))), // writer + Long[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getFloatArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toFloatPrimitive((Float[]) field.getField().get(obj))), // writer + Float[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, GenericsHelper.toObject(io.getDoubleArray())), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, GenericsHelper.toDoublePrimitive((Double[]) field.getField().get(obj))), // writer + Double[].class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldBoxedValueHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldBoxedValueHelper.java new file mode 100644 index 00000000..47eab5b0 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldBoxedValueHelper.java @@ -0,0 +1,68 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; + +/** + * helper class to register default serialiser for boxed primitive types (ie. Boolean, Byte, Short, ..., double) w/o + * String (already part of the {@link FieldPrimitiveValueHelper} + * + * @author rstein + */ +public final class FieldBoxedValueHelper { + public static final String NOT_SUPPORTED_FOR_PRIMITIVES = "return function not supported for primitive types"; + + private FieldBoxedValueHelper() { + // utility class + } + + /** + * registers default serialiser for primitive array types (ie. boolean[], byte[], short[], ..., double[]) and + * String[] + * + * @param serialiser for which the field serialisers should be registered + */ + public static void register(final IoClassSerialiser serialiser) { + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getBoolean()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Boolean) field.getField().get(obj)), // writer + Boolean.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getByte()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Byte) field.getField().get(obj)), // writer + Byte.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getShort()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Short) field.getField().get(obj)), // writer + Short.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getInt()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Integer) field.getField().get(obj)), // writer + Integer.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getLong()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Long) field.getField().get(obj)), // writer + Long.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getFloat()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Float) field.getField().get(obj)), // writer + Float.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getDouble()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (Double) field.getField().get(obj)), // writer + Double.class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldCollectionsHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldCollectionsHelper.java new file mode 100644 index 00000000..d117a340 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldCollectionsHelper.java @@ -0,0 +1,63 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import java.util.Collection; +import java.util.List; +import java.util.Queue; +import java.util.Set; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.utils.ClassUtils; + +public final class FieldCollectionsHelper { + private FieldCollectionsHelper() { + // utility class + } + + /** + * registers default Collection, List, Set, and Queue interface and related helper methods + * + * @param serialiser for which the field serialisers should be registered + */ + @SuppressWarnings("PMD.NPathComplexity") + public static void register(final IoClassSerialiser serialiser) { + // Collection serialiser mapper to IoBuffer + final FieldSerialiser.TriFunction> returnCollection = (io, obj, field) -> // + io.getCollection(field == null ? null : (Collection) field.getField().get(obj)); // return function + final FieldSerialiser.TriFunction> returnList = (io, obj, field) -> // + io.getList(field == null ? null : (List) field.getField().get(obj)); // return function + final FieldSerialiser.TriFunction> returnQueue = (io, obj, field) -> // + io.getQueue(field == null ? null : (Queue) field.getField().get(obj)); // return function + final FieldSerialiser.TriFunction> returnSet = (io, obj, field) -> // + io.getSet(field == null ? null : (Set) field.getField().get(obj)); // return function + + final FieldSerialiser.TriConsumer collectionWriter = (io, obj, field) -> { + if (field != null && !field.getActualTypeArguments().isEmpty() && ClassUtils.isPrimitiveWrapperOrString(ClassUtils.getRawType(field.getActualTypeArguments().get(0)))) { + io.put(field, (Collection) field.getField().get(obj), field.getActualTypeArguments().get(0)); + return; + } + if (field != null) { + // Collection serialiser + io.put(field, (Collection) field.getField().get(obj), field.getActualTypeArguments().get(0)); + return; + } + throw new IllegalArgumentException("serialiser for obj = '" + obj + "' and type = '" + (obj == null ? "null" : obj.getClass()) + "' not yet implemented, field = null"); + }; // writer + + serialiser.addClassDefinition(new FieldSerialiser<>((io, obj, field) -> // + field.getField().set(obj, returnCollection.apply(io, obj, field)), // reader + returnCollection, collectionWriter, Collection.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>((io, obj, field) -> // + field.getField().set(obj, returnList.apply(io, obj, field)), // reader + returnList, collectionWriter, List.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>((io, obj, field) -> // + field.getField().set(obj, returnQueue.apply(io, obj, field)), // reader + returnQueue, collectionWriter, Queue.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>((io, obj, field) -> // + field.getField().set(obj, returnSet.apply(io, obj, field)), // reader + returnSet, collectionWriter, Set.class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldDataSetHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldDataSetHelper.java new file mode 100644 index 00000000..e132d440 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldDataSetHelper.java @@ -0,0 +1,47 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; + +import de.gsi.dataset.DataSet; + +import it.unimi.dsi.fastutil.doubles.DoubleArrayList; + +public final class FieldDataSetHelper { + private FieldDataSetHelper() { + // utility class + } + + /** + * registers default DataSet interface and related helper methods + * + * @param serialiser for which the field serialisers should be registered + */ + public static void register(final IoClassSerialiser serialiser) { + // DoubleArrayList serialiser mapper to IoBuffer + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, DoubleArrayList.wrap(io.getDoubleArray())), // reader + (io, obj, field) -> DoubleArrayList.wrap(io.getDoubleArray()), // return + (io, obj, field) -> { + final DoubleArrayList retVal = (DoubleArrayList) field.getField().get(obj); + io.put(field, retVal.elements(), retVal.size()); + }, // writer + DoubleArrayList.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> { + // short form: FieldSerialiser.this.getReturnObjectFunction().andThen(io, obj, field) -- not possible inside a constructor + final DataSet origDataSet = (DataSet) field.getField().get(obj); + field.getField().set(obj, DataSetSerialiser.withIoSerialiser(io).read(origDataSet)); + }, // reader + (io, obj, field) -> DataSetSerialiser.withIoSerialiser(io).read(), // return object function + (io, obj, field) -> { + final DataSet origDataSet = (DataSet) (field == null || field.getField() == null ? obj : field.getField().get(obj)); + DataSetSerialiser.withIoSerialiser(io).write(origDataSet, false); + }, // writer + DataSet.class)); + + // List serialiser mapper to IoBuffer + serialiser.addClassDefinition(new FieldListAxisDescription()); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldListAxisDescription.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldListAxisDescription.java new file mode 100644 index 00000000..c4ef6cd2 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldListAxisDescription.java @@ -0,0 +1,76 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import java.util.ArrayList; +import java.util.List; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.ClassFieldDescription; + +import de.gsi.dataset.AxisDescription; +import de.gsi.dataset.spi.DefaultAxisDescription; + +/** + * FieldSerialiser implementation for List<AxisDescription> to IoBuffer-backed byte-buffer + * + * @author rstein + */ +public class FieldListAxisDescription extends FieldSerialiser> { + /** + * FieldSerialiser implementation for List<AxisDescription> to IoBuffer-backed byte-ioSerialiser + * + */ + public FieldListAxisDescription() { + super((io, obj, field) -> {}, (io, obj, field) -> null, (io, obj, field) -> {}, List.class, AxisDescription.class); + readerFunction = this::execFieldReader; + returnFunction = this::execFieldReturn; + writerFunction = this::execFieldWriter; + } + + protected void execFieldReader(final IoSerialiser ioSerialiser, final Object obj, ClassFieldDescription field) { + field.getField().set(obj, execFieldReturn(ioSerialiser, obj, field)); + } + + protected List execFieldReturn(final IoSerialiser ioSerialiser, Object obj, ClassFieldDescription field) { + final Object oldObject = field == null ? null : field.getField().get(obj); + final boolean isListPresent = oldObject instanceof List; + + final int nElements = ioSerialiser.getBuffer().getInt(); // number of elements + // N.B. cast should fail at runtime (points to lib inconsistency) + @SuppressWarnings("unchecked") + List setVal = isListPresent ? (List) field.getField().get(obj) : new ArrayList<>(nElements); // NOPMD + if (isListPresent) { + setVal.clear(); + } + + for (int i = 0; i < nElements; i++) { + String axisName = ioSerialiser.getBuffer().getString(); + String axisUnit = ioSerialiser.getBuffer().getString(); + double min = ioSerialiser.getBuffer().getDouble(); + double max = ioSerialiser.getBuffer().getDouble(); + DefaultAxisDescription ad = new DefaultAxisDescription(i, axisName, axisUnit, min, max); // NOPMD + // N.B. PMD - unavoidable in-loop instantiation + setVal.add(ad); + } + + return setVal; + } + + protected void execFieldWriter(final IoSerialiser ioSerialiser, Object obj, ClassFieldDescription field) { + @SuppressWarnings("unchecked") + final List axisDescriptions = (List) field.getField().get(obj); // NOPMD + // N.B. cast should fail at runtime (points to lib inconsistency) + + final int nElements = axisDescriptions.size(); + final int entrySize = 50; // as an initial estimate + + ioSerialiser.getBuffer().ensureAdditionalCapacity((nElements * entrySize) + 9); + ioSerialiser.getBuffer().putInt(nElements); // number of elements + for (AxisDescription axis : axisDescriptions) { + ioSerialiser.getBuffer().putString(axis.getName()); + ioSerialiser.getBuffer().putString(axis.getUnit()); + ioSerialiser.getBuffer().putDouble(axis.getMin()); + ioSerialiser.getBuffer().putDouble(axis.getMax()); + } + } +} \ No newline at end of file diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldMapHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldMapHelper.java new file mode 100644 index 00000000..062f6d41 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldMapHelper.java @@ -0,0 +1,32 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import java.util.Map; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; + +public final class FieldMapHelper { + private FieldMapHelper() { + // utility class + } + + /** + * registers default Map interface and related helper methods + * + * @param serialiser for which the field serialisers should be registered + */ + public static void register(final IoClassSerialiser serialiser) { + // Map serialiser mapper to IoBuffer + + final FieldSerialiser.TriFunction> returnMapFunction = (io, obj, field) -> { + final Map origMap = field == null ? null : (Map) field.getField().get(obj); + return io.getMap(origMap); + }; + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, returnMapFunction.apply(io, obj, field)), // reader + returnMapFunction, // return + (io, obj, field) -> io.put(field, (Map) field.getField().get(obj), field.getActualTypeArguments().get(0), field.getActualTypeArguments().get(1)), // writer + Map.class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldMultiArrayHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldMultiArrayHelper.java new file mode 100644 index 00000000..2d18e327 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldMultiArrayHelper.java @@ -0,0 +1,189 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +import de.gsi.dataset.spi.utils.*; + +/** + * helper class to register default serialiser for MultiArray types + * + * @author Alexander Krimm + */ +public final class FieldMultiArrayHelper { + private FieldMultiArrayHelper() { + // utility class + } + + @SuppressWarnings("unchecked") + public static MultiArray getMultiArray(final IoSerialiser serialiser, final MultiArray dst, final DataType type) { + final int[] dims = serialiser.getArraySizeDescriptor(); + int n = 1; + for (int ni : dims) { + n *= ni; + } + switch (type) { + case BOOL_ARRAY: + return (MultiArray) MultiArrayBoolean.wrap(serialiser.getBuffer().getBooleanArray(dst == null ? null : (boolean[]) dst.elements(), n), dims); + case BYTE_ARRAY: + return (MultiArray) MultiArrayByte.wrap(serialiser.getBuffer().getByteArray(dst == null ? null : (byte[]) dst.elements(), n), dims); + case SHORT_ARRAY: + return (MultiArray) MultiArrayShort.wrap(serialiser.getBuffer().getShortArray(dst == null ? null : (short[]) dst.elements(), n), dims); + case INT_ARRAY: + return (MultiArray) MultiArrayInt.wrap(serialiser.getBuffer().getIntArray(dst == null ? null : (int[]) dst.elements(), n), dims); + case LONG_ARRAY: + return (MultiArray) MultiArrayLong.wrap(serialiser.getBuffer().getLongArray(dst == null ? null : (long[]) dst.elements(), n), dims); + case FLOAT_ARRAY: + return (MultiArray) MultiArrayFloat.wrap(serialiser.getBuffer().getFloatArray(dst == null ? null : (float[]) dst.elements(), n), dims); + case DOUBLE_ARRAY: + return (MultiArray) MultiArrayDouble.wrap(serialiser.getBuffer().getDoubleArray(dst == null ? null : (double[]) dst.elements(), n), dims); + case CHAR_ARRAY: + return (MultiArray) MultiArrayChar.wrap(serialiser.getBuffer().getCharArray(dst == null ? null : (char[]) dst.elements(), n), dims); + case STRING_ARRAY: + return (MultiArray) MultiArrayObject.wrap(serialiser.getBuffer().getStringArray(dst == null ? null : (String[]) dst.elements(), n), dims); + default: + throw new IllegalStateException("Unexpected value: " + type); + } + } + + public static void put(final IoSerialiser serialiser, final String fieldName, final MultiArray value) { + if (value instanceof MultiArrayDouble) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.DOUBLE_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putDoubleArray(((MultiArrayDouble) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayFloat) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.FLOAT_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putFloatArray(((MultiArrayFloat) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayInt) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.INT_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putIntArray(((MultiArrayInt) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayLong) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.LONG_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putLongArray(((MultiArrayLong) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayShort) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.SHORT_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putShortArray(((MultiArrayShort) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayChar) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.CHAR_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putCharArray(((MultiArrayChar) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayByte) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.BYTE_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putByteArray(((MultiArrayByte) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else if (value instanceof MultiArrayObject) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldName, DataType.STRING_ARRAY); + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + serialiser.getBuffer().putStringArray(((MultiArrayObject) value).elements(), nElements); + serialiser.updateDataEndMarker(fieldHeader); + } else { + throw new IllegalArgumentException("Illegal DataType for MultiArray"); + } + } + + public static void put(final IoSerialiser serialiser, final FieldDescription fieldDescription, final MultiArray value) { + final WireDataFieldDescription fieldHeader = serialiser.putFieldHeader(fieldDescription); // NOPMD + final int nElements = serialiser.putArraySizeDescriptor(value.getDimensions()); + switch (fieldDescription.getDataType()) { + case BOOL_ARRAY: + serialiser.getBuffer().putBooleanArray(((MultiArrayBoolean) value).elements(), nElements); + break; + case BYTE_ARRAY: + serialiser.getBuffer().putByteArray(((MultiArrayByte) value).elements(), nElements); + break; + case SHORT_ARRAY: + serialiser.getBuffer().putShortArray(((MultiArrayShort) value).elements(), nElements); + break; + case INT_ARRAY: + serialiser.getBuffer().putIntArray(((MultiArrayInt) value).elements(), nElements); + break; + case LONG_ARRAY: + serialiser.getBuffer().putLongArray(((MultiArrayLong) value).elements(), nElements); + break; + case FLOAT_ARRAY: + serialiser.getBuffer().putFloatArray(((MultiArrayFloat) value).elements(), nElements); + break; + case DOUBLE_ARRAY: + serialiser.getBuffer().putDoubleArray(((MultiArrayDouble) value).elements(), nElements); + break; + case CHAR_ARRAY: + serialiser.getBuffer().putCharArray(((MultiArrayChar) value).elements(), nElements); + break; + case STRING_ARRAY: + serialiser.getBuffer().putStringArray(((MultiArrayObject) value).elements(), nElements); + break; + default: + throw new IllegalStateException("Unexpected value: " + fieldDescription.getDataType()); + } + serialiser.updateDataEndMarker(fieldHeader); + } + + /** + * Registers default serialiser for MultiArray + * + * @param serialiser for which the field serialisers should be registered + */ + @SuppressWarnings("PMD.NPathComplexity") + public static void register(final IoClassSerialiser serialiser) { + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.DOUBLE_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayDouble.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.FLOAT_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayFloat.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.INT_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayInt.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.LONG_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayLong.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.SHORT_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayShort.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.BYTE_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayByte.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.CHAR_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayChar.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.BOOL_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayBoolean.class)); + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, getMultiArray(io, (MultiArray) field.getField().get(obj), field.getDataType())), // reader + (io, obj, field) -> getMultiArray(io, (MultiArray) ((field == null) ? obj : field.getField().get(obj)), DataType.STRING_ARRAY), // return + (io, obj, field) -> put(io, field, (MultiArray) field.getField().get(obj)), // writer + MultiArrayObject.class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldPrimitiveValueHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldPrimitiveValueHelper.java new file mode 100644 index 00000000..a354f2f8 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldPrimitiveValueHelper.java @@ -0,0 +1,78 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; + +/** + * helper class to register default serialiser for primitive types (ie. boolean, byte, short, ..., double) and String + * + * @author rstein + */ +public final class FieldPrimitiveValueHelper { + public static final String UNSUPPORTED = "return function not supported for primitive types"; + + private FieldPrimitiveValueHelper() { + // utility class + } + + /** + * registers default serialiser for primitive types (ie. boolean, byte, short, ..., double) and String + * + * @param serialiser for which the field serialisers should be registered + */ + public static void register(final IoClassSerialiser serialiser) { + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setBoolean(obj, io.getBoolean()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getBoolean(obj)), // writer + boolean.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setByte(obj, io.getByte()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getByte(obj)), // writer + byte.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setChar(obj, io.getChar()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getChar(obj)), // writer + char.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setShort(obj, io.getShort()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getShort(obj)), // writer + short.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setInt(obj, io.getInt()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getInt(obj)), // writer + int.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setLong(obj, io.getLong()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getLong(obj)), // writer + long.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setFloat(obj, io.getFloat()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getFloat(obj)), // writer + float.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().setDouble(obj, io.getDouble()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, field.getField().getDouble(obj)), // writer + double.class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getString()), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(UNSUPPORTED); }, // return + (io, obj, field) -> io.put(field, (String) field.getField().get(obj)), // writer + String.class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldPrimitveValueArrayHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldPrimitveValueArrayHelper.java new file mode 100644 index 00000000..27f10390 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/spi/iobuffer/FieldPrimitveValueArrayHelper.java @@ -0,0 +1,80 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import io.opencmw.serialiser.FieldSerialiser; +import io.opencmw.serialiser.IoClassSerialiser; + +/** + * helper class to register default serialiser for primitive array types (ie. boolean[], byte[], short[], ..., double[]) + * and String[] + * + * @author rstein + */ +public final class FieldPrimitveValueArrayHelper { + public static final String NOT_SUPPORTED_FOR_PRIMITIVES = "return function not supported for primitive types"; + + private FieldPrimitveValueArrayHelper() { + // utility class + } + + /** + * registers default serialiser for array primitive types (ie. boolean[], byte[], short[], ..., double[]) and + * String[] + * + * @param serialiser for which the field serialisers should be registered + */ + public static void register(final IoClassSerialiser serialiser) { + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getBooleanArray((boolean[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (boolean[]) field.getField().get(obj)), // writer + boolean[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getByteArray((byte[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (byte[]) field.getField().get(obj)), // writer + byte[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getCharArray((char[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (char[]) field.getField().get(obj)), // writer + char[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getShortArray((short[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (short[]) field.getField().get(obj)), // writer + short[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getIntArray((int[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (int[]) field.getField().get(obj)), // writer + int[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getLongArray((long[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (long[]) field.getField().get(obj)), // writer + long[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getFloatArray((float[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (float[]) field.getField().get(obj)), // writer + float[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getDoubleArray((double[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (double[]) field.getField().get(obj)), // writer + double[].class)); + + serialiser.addClassDefinition(new FieldSerialiser<>( // + (io, obj, field) -> field.getField().set(obj, io.getStringArray((String[]) field.getField().get(obj))), // reader + (io, obj, field) -> { throw new UnsupportedOperationException(NOT_SUPPORTED_FOR_PRIMITIVES); }, // return + (io, obj, field) -> io.put(field, (String[]) field.getField().get(obj)), // writer + String[].class)); + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/utils/AssertUtils.java b/serialiser/src/main/java/io/opencmw/serialiser/utils/AssertUtils.java new file mode 100644 index 00000000..4e6b1329 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/utils/AssertUtils.java @@ -0,0 +1,393 @@ +package io.opencmw.serialiser.utils; + +/** + * Utility class used to examine function parameters. All the methods throw IllegalArgumentException if the + * argument doesn't fulfil constraints. + * + * @author rstein + */ +public final class AssertUtils { + private static final String MUST_BE_GREATER_THAN_OR_EQUAL_TO_0 = " must be greater than or equal to 0!"; + private static final String MUST_BE_NON_EMPTY = " must be non-empty!"; + + private AssertUtils() { + } + + /** + * The method returns true if both values area equal. The method differs from simple == compare because it takes + * into account that both values can be Double.NaN, in which case == operator returns false. + * + * @param v1 to be checked + * @param v2 to be checked + * + * @return true if v1 and v2 are Double.NaN or v1 == v2. + */ + public static boolean areEqual(final double v1, final double v2) { + return (Double.isNaN(v1) && Double.isNaN(v2)) || (v1 == v2); + } + + /** + * Asserts if the specified object is an instance of the specified type. + * + * @param obj to be checked + * @param type required class type + * + * @throws IllegalArgumentException in case of problems + */ + public static void assertType(final Object obj, final Class type) { + if (!type.isInstance(obj)) { + throw new IllegalArgumentException( + "The argument has incorrect type. The correct type is " + type.getName()); + } + } + + public static void belongsToEnum(final String name, final int[] allowedElements, final int value) { + for (final int allowedElement : allowedElements) { + if (value == allowedElement) { + return; + } + } + throw new IllegalArgumentException("The " + name + " has incorrect value!"); + } + + public static void checkArrayDimension(final String name, final boolean[] array, final int defaultLength) { + AssertUtils.notNull(name, array); + AssertUtils.nonEmptyArray(name, array); + if (array.length != defaultLength) { + throw new IllegalArgumentException("The " + name + " boolean array must have a length of " + defaultLength); + } + } + + public static void checkArrayDimension(final String name, final byte[] array, final int defaultLength) { + AssertUtils.notNull(name, array); + AssertUtils.nonEmptyArray(name, array); + if (array.length != defaultLength) { + throw new IllegalArgumentException("The " + name + " byte array must have a length of " + defaultLength); + } + } + + public static void checkArrayDimension(final String name, final double[] array, final int defaultLength) { + AssertUtils.notNull(name, array); + AssertUtils.nonEmptyArray(name, array); + if (array.length != defaultLength) { + throw new IllegalArgumentException("The " + name + " double array must have a length of " + defaultLength); + } + } + + public static void checkArrayDimension(final String name, final float[] array, final int defaultLength) { + AssertUtils.notNull(name, array); + AssertUtils.nonEmptyArray(name, array); + if (array.length != defaultLength) { + throw new IllegalArgumentException("The " + name + " float array must have a length of " + defaultLength); + } + } + + public static void checkArrayDimension(final String name, final int[] array, final int defaultLength) { + AssertUtils.notNull(name, array); + AssertUtils.nonEmptyArray(name, array); + if (array.length != defaultLength) { + throw new IllegalArgumentException("The " + name + " int array must have a length of " + defaultLength); + } + } + + /** + * Asserts that the specified arrays have the same length. + * + * @param generics object to be checked + * + * @param array1 to be checked + * @param array2 to be checked + */ + public static void equalArrays(final T[] array1, final T[] array2) { + if (array1.length != array2.length) { + throw new IllegalArgumentException("The double arrays must have the same length!"); + } + } + + /** + * Asserts that the specified arrays have the same length. + * + * @param array1 to be checked + * @param array2 to be checked + */ + public static void equalDoubleArrays(final double[] array1, final double[] array2) { + if (array1.length != array2.length) { + throw new IllegalArgumentException("The double arrays must have the same length! length1 = " + array1.length + + " vs. length2 = " + array2.length); + } + } + + /** + * Asserts that the specified arrays have the same length or are at least min size. + * + * @param array1 to be checked + * @param array2 to be checked + * @param nMinSize minimum required size + */ + public static void equalDoubleArrays(final double[] array1, final double[] array2, final int nMinSize) { + final int length1 = Math.min(nMinSize, array1.length); + final int length2 = Math.min(nMinSize, array2.length); + if (length1 != length2) { + throw new IllegalArgumentException("The double arrays must have the same length! length1 = " + array1.length + + " vs. length2 = " + array2.length + " (nMinSize = " + nMinSize + ")"); + } + } + + /** + * Asserts that the specified arrays have the same length. + * + * @param array1 to be checked + * @param array2 to be checked + */ + public static void equalFloatArrays(final float[] array1, final float[] array2) { + if (array1.length != array2.length) { + throw new IllegalArgumentException("The float arrays must have the same length! length1 = " + array1.length + + " vs. length2 = " + array2.length); + } + } + + /** + * Asserts that the specified arrays have the same length or are at least min size. + * + * @param array1 to be checked + * @param array2 to be checked + * @param nMinSize minimum required size + */ + public static void equalFloatArrays(final float[] array1, final float[] array2, final int nMinSize) { + final int length1 = Math.min(nMinSize, array1.length); + final int length2 = Math.min(nMinSize, array2.length); + if (length1 != length2) { + throw new IllegalArgumentException("The double arrays must have the same length! length1 = " + array1.length + + " vs. length2 = " + array2.length + " (nMinSize = " + nMinSize + ")"); + } + } + + /** + * Checks if the int value is >= 0 + * + * @param name name to be included in the exception message + * @param value to be checked + */ + public static void gtEqThanZero(final String name, final double value) { + if (value < 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_GREATER_THAN_OR_EQUAL_TO_0); + } + } + + /** + * Checks if the int value is >= 0 + * + * @param name name to be included in the exception message + * @param value to be checked + */ + public static void gtEqThanZero(final String name, final int value) { + if (value < 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_GREATER_THAN_OR_EQUAL_TO_0); + } + } + + /** + * Checks if the value is >= 0 + * + * @param generics object to be checked + * + * @param name name to be included in the exception message + * @param value to be checked + */ + public static void gtEqThanZero(final String name, final T value) { + if (value.doubleValue() < 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_GREATER_THAN_OR_EQUAL_TO_0); + } + } + + /** + * Checks if the int value is >= 0 + * + * @param name name to be included in the exception message + * @param value to be checked + */ + public static void gtThanZero(final String name, final int value) { + if (value <= 0) { + throw new IllegalArgumentException("The " + name + " must be greater than 0!"); + } + } + + /** + * Checks if the value is >= 0 + * + * @param generics object to be checked + * + * @param name name to be included in the exception message + * @param value to be checked + */ + public static void gtThanZero(final String name, final T value) { + if (value.doubleValue() <= 0) { + throw new IllegalArgumentException("The " + name + " must be greater than 0!"); + } + } + + /** + * Checks if the index is >= 0 and < bounds + * + * @param index index to be checked + * @param bounds maximum bound + */ + public static void indexInBounds(final int index, final int bounds) { + AssertUtils.indexInBounds(index, bounds, "The index is out of bounds: 0 <= " + index + " < " + bounds); + } + + /** + * Checks if the index is >= 0 and < bounds + * + * @param index index to be checked + * @param bounds maximum bound + * @param message exception message + */ + public static void indexInBounds(final int index, final int bounds, final String message) { + if ((index < 0) || (index >= bounds)) { + throw new IndexOutOfBoundsException(message); + } + } + + /** + * Checks if the index1 <= index2 + * + * @param index1 index1 to be checked + * @param index2 index1 to be checked + * @param msg exception message + */ + public static void indexOrder(final int index1, final int index2, final String msg) { + if (index1 > index2) { + throw new IndexOutOfBoundsException(msg); + } + } + + /** + * Checks if the index1 <= index2 + * + * @param index1 index1 to be checked + * @param name1 name of index1 + * @param index2 index1 to be checked + * @param name2 name of index2 + */ + public static void indexOrder(final int index1, final String name1, final int index2, final String name2) { + if (index1 > index2) { + throw new IndexOutOfBoundsException( + "Index " + name1 + "(" + index1 + ") is greated than index " + name2 + "(" + index2 + ")"); + } + } + + /** + * Checks if the variable is less or equal than the reference + * + * @param name name to be included in exception message. + * @param ref reference + * @param len object to be checked + */ + public static void gtOrEqual(final String name, final double ref, final double len) { + if (len < ref) { + throw new IllegalArgumentException("The " + name + " len = '" + len + "' must be less or equal than " + ref); + } + } + + /** + * Checks if the variable is less or equal than the reference + * + * @param name name to be included in exception message. + * @param ref reference + * @param len object to be checked + */ + public static void gtOrEqual(final String name, final float ref, final float len) { + if (len < ref) { + throw new IllegalArgumentException("The " + name + " len = '" + len + "' must be less or equal than " + ref); + } + } + + /** + * Checks if the variable is greater or equal than the reference + * + * @param name name to be included in exception message. + * @param ref reference + * @param len object to be checked + */ + public static void gtOrEqual(final String name, final int ref, final int len) { + if (len < ref) { + throw new IllegalArgumentException("The " + name + " len = '" + len + "' must be greater or equal than " + ref); + } + } + + /** + * Checks if the variable is less or equal than the reference + * + * @param name name to be included in exception message. + * @param ref reference + * @param len object to be checked + */ + public static void gtOrEqual(final String name, final long ref, final long len) { + if (len < ref) { + throw new IllegalArgumentException("The " + name + " len = '" + len + "' must be less or equal than " + ref); + } + } + + public static void nonEmptyArray(final String name, final boolean[] array) { + AssertUtils.notNull(name, array); + if (array.length == 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_NON_EMPTY); + } + } + + public static void nonEmptyArray(final String name, final byte[] array) { + AssertUtils.notNull(name, array); + if (array.length == 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_NON_EMPTY); + } + } + + public static void nonEmptyArray(final String name, final double[] array) { + AssertUtils.notNull(name, array); + if (array.length == 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_NON_EMPTY); + } + } + + public static void nonEmptyArray(final String name, final float[] array) { + AssertUtils.notNull(name, array); + if (array.length == 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_NON_EMPTY); + } + } + + public static void nonEmptyArray(final String name, final int[] array) { + AssertUtils.notNull(name, array); + if (array.length == 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_NON_EMPTY); + } + } + + public static void nonEmptyArray(final String name, final Object[] array) { + AssertUtils.notNull(name, array); + if (array.length == 0) { + throw new IllegalArgumentException("The " + name + MUST_BE_NON_EMPTY); + } + + for (final Object element : array) { + if (element == null) { + throw new NullPointerException("Elements of the " + name + " must be non-null!"); // #NOPMD + } + } + } + + /** + * Checks if the object is not null. + * + * @param generics object to be checked + * + * @param name name to be included in exception message. + * @param obj object to be checked + */ + public static void notNull(final String name, final T obj) { + if (obj == null) { + throw new IllegalArgumentException("The " + name + " must be non-null!"); + } + } +} \ No newline at end of file diff --git a/serialiser/src/main/java/io/opencmw/serialiser/utils/ByteArrayCache.java b/serialiser/src/main/java/io/opencmw/serialiser/utils/ByteArrayCache.java new file mode 100644 index 00000000..9888c090 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/utils/ByteArrayCache.java @@ -0,0 +1,85 @@ +package io.opencmw.serialiser.utils; + +import java.lang.ref.Reference; + +/** + * Implements byte-array (byte[]) cache collection to minimise memory re-allocation. + * + *

+ * N.B. This is useful to re-use short-lived data storage container to minimise the amount of garbage to be collected. This is used in situation replacing e.g. + * 

+ *  {@code
+ *      public returnValue frequentlyExecutedFunction(...) {
+ *          final byte[] storage = new byte[10000]; // allocate new memory block (costly)
+ *          // [...] do short-lived computation on storage
+ *          // storage is implicitly finalised by garbage collector (costly)
+ *      }
+ *  }
+ *
+ * with + * 
+ *  {@code
+ *      // ...
+ *      private final ByteArrayCache cache = new ByteArrayCache();
+ *      // ...
+ *      
+ *      public returnValue frequentlyExecutedFunction(...) {
+ *          final byte[] storage = cache.getArray(10000); // return previously allocated array (cheap) or allocated new if necessary 
+ *          // [...] do short-lived computation on storage
+ *          cache.add(storage); // return object to cache
+ *      }
+ *  }
+ *
+ * + * @author rstein + * + */ +public class ByteArrayCache extends CacheCollection { + private static final ByteArrayCache SELF = new ByteArrayCache(); + + public byte[] getArray(final int requiredSize) { + return getArray(requiredSize, false); + } + + public byte[] getArrayExact(final int requiredSize) { + return getArray(requiredSize, true); + } + + private byte[] getArray(final int requiredSize, final boolean exact) { + synchronized (contents) { + byte[] bestFit = null; + int bestFitSize = Integer.MAX_VALUE; + + for (final Reference candidate : contents) { + final byte[] localRef = candidate.get(); + if (localRef == null) { + continue; + } + + final int sizeDiff = localRef.length - requiredSize; + if (sizeDiff == 0) { + bestFit = localRef; + break; + } + + if (sizeDiff > 0 && sizeDiff < bestFitSize && !exact) { + bestFitSize = sizeDiff; + bestFit = localRef; + } + } + + if (bestFit == null) { + // could not find any cached, return new byte[] + bestFit = new byte[requiredSize]; + return bestFit; + } + + remove(bestFit); + return bestFit; + } + } + + public static ByteArrayCache getInstance() { + return SELF; + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/utils/CacheCollection.java b/serialiser/src/main/java/io/opencmw/serialiser/utils/CacheCollection.java new file mode 100644 index 00000000..37dbd094 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/utils/CacheCollection.java @@ -0,0 +1,153 @@ +package io.opencmw.serialiser.utils; + +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.util.AbstractCollection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +import org.jetbrains.annotations.NotNull; + +import de.gsi.dataset.utils.ByteArrayCache; + +/** + * Implements collection of cache-able objects that can be used to store recurring storage container. + *

+ * N.B. this implements only the backing cache of adding, removing, etc. elements. The cache object retrieval should be implemented in the derived class. + * See for example {@link ByteArrayCache}. + * + * @author rstein + * + * @param generic for object type to be cached. + */ +public class CacheCollection extends AbstractCollection { + protected final List> contents = Collections.synchronizedList(new LinkedList<>()); + + @Override + public boolean add(T recoveredObject) { + if (recoveredObject != null) { + synchronized (contents) { + if (contains(recoveredObject)) { + return false; + } + // N.B. here: specific choice of using 'SoftReference' + // derived classes may overwrite this function and replace this with e.g. WeakReference or similar + return contents.add(new SoftReference<>(recoveredObject)); + } + } + return false; + } + + @Override + public void clear() { + synchronized (contents) { + contents.clear(); + } + } + + @Override + public boolean contains(Object object) { + if (object != null) { + synchronized (contents) { + for (Reference weakReference : contents) { + if (object.equals(weakReference.get())) { + return true; + } + } + } + } + return false; + } + + @Override + public @NotNull Iterator iterator() { + synchronized (contents) { + return new CacheCollectionIterator<>(contents.iterator()); + } + } + + @Override + public boolean remove(Object o) { + if (o == null) { + return false; + } + synchronized (contents) { + Iterator> iter = contents.iterator(); + while (iter.hasNext()) { + final Reference candidate = iter.next(); + final T test = candidate.get(); + if (test == null) { + iter.remove(); + continue; + } + if (o.equals(test)) { + iter.remove(); + return true; + } + } + } + return false; + } + + @Override + public int size() { + synchronized (contents) { + cleanup(); + return contents.size(); + } + } + + protected void cleanup() { + synchronized (contents) { + List> toRemove = new LinkedList<>(); + for (Reference weakReference : contents) { + if (weakReference.get() == null) { + toRemove.add(weakReference); + } + } + contents.removeAll(toRemove); + } + } + + private static class CacheCollectionIterator implements Iterator { + private final Iterator> iterator; + private T nextElement; + + private CacheCollectionIterator(Iterator> iterator) { + this.iterator = iterator; + } + + @Override + public boolean hasNext() { + if (nextElement != null) { + return true; + } + while (iterator.hasNext()) { + T t = iterator.next().get(); + if (t != null) { + // to ensure next() can't throw after hasNext() returned true, we need to dereference this + nextElement = t; + return true; + } + } + return false; + } + + @Override + public T next() { + T result = nextElement; + nextElement = null; // NOPMD + while (result == null) { + result = iterator.next().get(); + } + return result; + } + + @Override + public void remove() { + iterator.remove(); + } + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/utils/ClassUtils.java b/serialiser/src/main/java/io/opencmw/serialiser/utils/ClassUtils.java new file mode 100644 index 00000000..8a8d32a6 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/utils/ClassUtils.java @@ -0,0 +1,271 @@ +package io.opencmw.serialiser.utils; + +import java.lang.reflect.Array; +import java.lang.reflect.GenericArrayType; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.lang.reflect.WildcardType; +import java.nio.CharBuffer; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.spi.ClassFieldDescription; + +@SuppressWarnings("PMD.UseConcurrentHashMap") +public final class ClassUtils { //NOPMD nomen est omen + private static final Logger LOGGER = LoggerFactory.getLogger(ClassUtils.class); + // some helper declarations + private static final Map, Class> PRIMITIVE_WRAPPER_MAP = new HashMap<>(); + private static final Map, Class> WRAPPER_PRIMITIVE_MAP = new HashMap<>(); + private static final Map, Class> PRIMITIVE_ARRAY_BOXED_MAP = new HashMap<>(); + private static final Map, Class> BOXED_ARRAY_PRIMITIVE_MAP = new HashMap<>(); + public static final Map, String> DO_NOT_PARSE_MAP = new HashMap<>(); // NOPMD should not be a threading issue - static one-time/init write, multiple reads afterwards are safe + private static final Map CLASS_FIELD_DESCRIPTION_MAP = new ConcurrentHashMap<>(); + private static final Map> CLASS_STRING_MAP = new ConcurrentHashMap<>(); + private static final Map, Map> CLASS_METHOD_MAP = new ConcurrentHashMap<>(); + private static int indentationNumberOfSpace = 4; + private static int maxRecursionDepth = 10; + + static { + // primitive types + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Boolean.class, boolean.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Byte.class, byte.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Character.class, char.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Short.class, short.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Integer.class, int.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Long.class, long.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Float.class, float.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Double.class, double.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, Void.class, void.class); + add(WRAPPER_PRIMITIVE_MAP, PRIMITIVE_WRAPPER_MAP, String.class, String.class); + + // primitive arrays + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, boolean[].class, Boolean[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, byte[].class, Byte[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, char[].class, Character[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, short[].class, Short[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, int[].class, Integer[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, long[].class, Long[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, float[].class, Float[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, double[].class, Double[].class); + add(PRIMITIVE_ARRAY_BOXED_MAP, BOXED_ARRAY_PRIMITIVE_MAP, String[].class, String[].class); + + // boxed arrays + + // do not parse following classes + DO_NOT_PARSE_MAP.put(Class.class, "private java implementation"); + DO_NOT_PARSE_MAP.put(Thread.class, "recursive definitions"); // NOPMD - not an issue/not a use within a J2EE context + DO_NOT_PARSE_MAP.put(AtomicBoolean.class, "does not like to be parsed"); + } + private ClassUtils() { + // utility class + } + + public static void checkArgument(boolean condition) { + if (!condition) { + throw new IllegalArgumentException(); + } + } + + public static Class getClassByName(final String name) { + return CLASS_STRING_MAP.computeIfAbsent(name, key -> { + try { + return Class.forName(key); + } catch (ClassNotFoundException | SecurityException e) { + LOGGER.atError().setCause(e).addArgument(name).log("exception while getting class {}"); + return null; + } + }); + } + + public static Class getClassByNameNonVerboseError(final String name) { + return CLASS_STRING_MAP.computeIfAbsent(name, key -> { + try { + return Class.forName(key); + } catch (ClassNotFoundException | SecurityException e) { + return Object.class; + } + }); + } + + public static Map getClassDescriptions() { + return CLASS_FIELD_DESCRIPTION_MAP; + } + + public static ClassFieldDescription getFieldDescription(final Class clazz, final Class... classArguments) { + if (clazz == null) { + throw new IllegalArgumentException("object must not be null"); + } + return CLASS_FIELD_DESCRIPTION_MAP.computeIfAbsent(computeHashCode(clazz, classArguments), + key -> new ClassFieldDescription(clazz, false)); + } + + public static int getIndentationNumberOfSpace() { + return indentationNumberOfSpace; + } + + public static Collection getKnownClasses() { + return CLASS_FIELD_DESCRIPTION_MAP.values(); + } + + public static Map, Map> getKnownMethods() { + return CLASS_METHOD_MAP; + } + + public static int getMaxRecursionDepth() { + return maxRecursionDepth; + } + + public static Method getMethod(final Class clazz, final String methodName) { + return CLASS_METHOD_MAP.computeIfAbsent(clazz, c -> new ConcurrentHashMap<>()).computeIfAbsent(methodName, name -> { + try { + return clazz.getMethod(methodName); + } catch (NoSuchMethodException | SecurityException e) { + return null; + } + }); + } + + public static Class getRawType(Type type) { + if (type instanceof Class) { + // type is a normal class. + return (Class) type; + + } else if (type instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType) type; + + // unsure why getRawType() returns Type instead of Class. + // possibly related to pathological case involving nested classes.... + Type rawType = parameterizedType.getRawType(); + checkArgument(rawType instanceof Class); + return (Class) rawType; + + } else if (type instanceof GenericArrayType) { + Type componentType = ((GenericArrayType) type).getGenericComponentType(); + return Array.newInstance(getRawType(componentType), 0).getClass(); + + } else if (type instanceof TypeVariable) { + // we could use the variable's bounds, but that won't work if there are multiple. + // having a raw type that's more general than necessary is okay + return Object.class; + + } else if (type instanceof WildcardType) { + return getRawType(((WildcardType) type).getUpperBounds()[0]); + + } else { + String className = type == null ? "null" : type.getClass().getName(); + throw new IllegalArgumentException("Expected a Class, ParameterizedType, or " + + "GenericArrayType, but <" + type + "> is of type " + className); + } + } + + public static Type[] getSecondaryType(final Type type) { + if (type instanceof ParameterizedType) { + return ((ParameterizedType) type).getActualTypeArguments(); + } + return new Type[0]; + } + + public static boolean isBoxedArray(final Class type) { + return BOXED_ARRAY_PRIMITIVE_MAP.containsKey(type); + } + + public static boolean isPrimitiveArray(final Class type) { + return PRIMITIVE_ARRAY_BOXED_MAP.containsKey(type); + } + + public static boolean isPrimitiveOrString(final Class type) { + if (type == null) { + return false; + } + return type.isPrimitive() || String.class.isAssignableFrom(type); + } + + public static boolean isPrimitiveOrWrapper(final Class type) { + if (type == null) { + return false; + } + return type.isPrimitive() || isPrimitiveWrapper(type); + } + + public static boolean isPrimitiveWrapper(final Class type) { + return WRAPPER_PRIMITIVE_MAP.containsKey(type); + } + + public static boolean isPrimitiveWrapperOrString(final Class type) { + if (type == null) { + return false; + } + return WRAPPER_PRIMITIVE_MAP.containsKey(type) || String.class.isAssignableFrom(type); + } + + public static Class primitiveToWrapper(final Class cls) { + Class convertedClass = cls; + if (cls != null && cls.isPrimitive()) { + convertedClass = PRIMITIVE_WRAPPER_MAP.get(cls); + } + return convertedClass; + } + + public static void setIndentationNumberOfSpace(final int indentationNumberOfSpace) { + ClassUtils.indentationNumberOfSpace = indentationNumberOfSpace; + } + + public static void setMaxRecursionDepth(final int maxRecursionDepth) { + ClassUtils.maxRecursionDepth = maxRecursionDepth; + } + + public static String spaces(final int spaces) { + return CharBuffer.allocate(spaces).toString().replace('\0', ' '); + } + + public static String translateClassName(final String name) { + if (name.startsWith("[Z")) { + return boolean[].class.getName(); + } else if (name.startsWith("[B")) { + return byte[].class.getName(); + } else if (name.startsWith("[S")) { + return short[].class.getName(); + } else if (name.startsWith("[I")) { + return int[].class.getName(); + } else if (name.startsWith("[J")) { + return long[].class.getName(); + } else if (name.startsWith("[F")) { + return float[].class.getName(); + } else if (name.startsWith("[D")) { + return double[].class.getName(); + } else if (name.startsWith("[L")) { + return name.substring(2, name.length() - 1) + "[]"; + } + + return name; + } + + private static void add(Map, Class> map1, Map, Class> map2, Class obj1, Class obj2) { + map1.put(obj1, obj2); + map2.put(obj2, obj1); + } + + private static int computeHashCode(final Class classPrototype, final Class... classArguments) { + final int prime = 31; + int result = 1; + result = (prime * result) + ((classPrototype == null) ? 0 : classPrototype.getName().hashCode()); + if ((classArguments == null) || (classArguments.length <= 0)) { + return result; + } + + for (final Class clazz : classArguments) { + result = (prime * result) + clazz.hashCode(); + } + + return result; + } +} diff --git a/serialiser/src/main/java/io/opencmw/serialiser/utils/GenericsHelper.java b/serialiser/src/main/java/io/opencmw/serialiser/utils/GenericsHelper.java new file mode 100644 index 00000000..a8b16a65 --- /dev/null +++ b/serialiser/src/main/java/io/opencmw/serialiser/utils/GenericsHelper.java @@ -0,0 +1,325 @@ +package io.opencmw.serialiser.utils; + +/** + * Helper class to convert between boxed and primitive data types. Lot's of boiler plate code because java generics + * cannot handle primitive types. + * + * @author rstein + */ +@SuppressWarnings({ "PMD.GodClass", "PMD.TooManyMethods", "PMD.ExcessivePublicCount", "PMD.AvoidArrayLoops" }) // unavoidable since Java does not support templates (issue: primitive types) +public final class GenericsHelper { + private GenericsHelper() { + // only static methods are used + } + + private static void checkForNonConformity(Object[] array, Class prototype) { + if (array == null) { + throw new IllegalArgumentException("null array pointer "); + } + + if (array.length == 0) { + return; + } + + for (int i = 0; i < array.length; i++) { + if (array[i] == null) { + throw new IllegalArgumentException( + "array class element " + i + " is null, should be of type'" + prototype.getName() + "'"); + } + } + + if (!prototype.isAssignableFrom(array[0].getClass())) { + throw new IllegalArgumentException("array class type '" + array[0].getClass().getName() + + "' mismatch with '" + prototype.getName() + "'"); + } + } + + public static boolean[] toBoolPrimitive(final Object[] array) { + checkForNonConformity(array, Boolean.class); + final boolean[] result = new boolean[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] != null && (Boolean) array[i]; + } + return result; + } + + public static byte[] toBytePrimitive(final Object[] array) { + checkForNonConformity(array, Byte.class); + final byte[] result = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] == null ? 0 : (Byte) array[i]; + } + return result; + } + + public static char[] toCharPrimitive(final Object[] array) { + checkForNonConformity(array, Character.class); + final char[] result = new char[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] == null ? 0 : (Character) array[i]; + } + return result; + } + + public static double[] toDoublePrimitive(final Object[] array) { + checkForNonConformity(array, Double.class); + final double[] result = new double[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] == null ? Double.NaN : (Double) array[i]; + } + return result; + } + + public static float[] toFloatPrimitive(final Object[] array) { + checkForNonConformity(array, Float.class); + final float[] result = new float[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] == null ? Float.NaN : (Float) array[i]; + } + return result; + } + + public static int[] toIntegerPrimitive(final Object[] array) { + checkForNonConformity(array, Integer.class); + final int[] result = new int[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] == null ? 0 : (Integer) array[i]; + } + return result; + } + + public static long[] toLongPrimitive(final Object[] array) { + checkForNonConformity(array, Long.class); + final long[] result = new long[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i] == null ? 0L : (Long) array[i]; + } + return result; + } + + public static Boolean[] toObject(final boolean[] array) { + final Boolean[] result = new Boolean[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Byte[] toObject(final byte[] array) { + final Byte[] result = new Byte[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Character[] toObject(final char[] array) { + final Character[] result = new Character[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Double[] toObject(final double[] array) { + final Double[] result = new Double[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Float[] toObject(final float[] array) { + final Float[] result = new Float[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Integer[] toObject(final int[] array) { + final Integer[] result = new Integer[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Long[] toObject(final long[] array) { + final Long[] result = new Long[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static Short[] toObject(final short[] array) { // NOPMD + + final Short[] result = new Short[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static boolean[] toPrimitive(final Boolean[] array) { + final boolean[] result = new boolean[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static byte[] toPrimitive(final Byte[] array) { + final byte[] result = new byte[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static char[] toPrimitive(final Character[] array) { + final char[] result = new char[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static double[] toPrimitive(final Double[] array) { + final double[] result = new double[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static float[] toPrimitive(final Float[] array) { + final float[] result = new float[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static int[] toPrimitive(final Integer[] array) { + final int[] result = new int[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static long[] toPrimitive(final Long[] array) { + final long[] result = new long[array.length]; + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static short[] toPrimitive(final Short[] array) { // NOPMD + final short[] result = new short[array.length]; // NOPMD + for (int i = 0; i < array.length; i++) { + result[i] = array[i]; + } + return result; + } + + public static short[] toShortPrimitive(final Object[] array) { // NOPMD + final short[] result = new short[array.length]; // NOPMD + for (int i = 0; i < array.length; i++) { + result[i] = (Short) array[i]; + } + return result; + } + + public static String[] toStringPrimitive(final Object[] array) { + final String[] result = new String[array.length]; + if (array.length == 0) { + return result; + } + if (array[0] instanceof String) { + for (int i = 0; i < array.length; i++) { + result[i] = (String) array[i]; + } + } else if (array[0] instanceof Boolean) { + for (int i = 0; i < array.length; i++) { + result[i] = ((Boolean) array[i]).toString(); + } + } else if (array[0] instanceof Character) { + for (int i = 0; i < array.length; i++) { + result[i] = array[i].toString(); + } + } else if (array[0] instanceof Number) { + for (int i = 0; i < array.length; i++) { + result[i] = array[i].toString(); + } + } + return result; + } + + public static double[] toDoublePrimitive(final boolean[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i] ? 1.0 : 0.0; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final byte[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i]; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final char[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i]; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final float[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i]; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final int[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i]; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final long[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i]; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final short[] input) { // NOPMD + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i]; + } + return doubleArray; + } + + public static double[] toDoublePrimitive(final String[] input) { + final double[] doubleArray = new double[input.length]; + for (int i = 0; i < input.length; i++) { // NOPMD + doubleArray[i] = input[i] == null ? Double.NaN : Double.parseDouble(input[i]); + } + return doubleArray; + } +} diff --git a/serialiser/src/main/java/module-info.java b/serialiser/src/main/java/module-info.java new file mode 100644 index 00000000..a5de31ea --- /dev/null +++ b/serialiser/src/main/java/module-info.java @@ -0,0 +1,14 @@ +module io.opencmw.serialiser { + requires static de.gsi.chartfx.dataset; + requires org.slf4j; + requires jsoniter; + requires jdk.unsupported; + requires it.unimi.dsi.fastutil; + requires org.jetbrains.annotations; + + exports io.opencmw.serialiser; + exports io.opencmw.serialiser.annotations; + exports io.opencmw.serialiser.spi; + exports io.opencmw.serialiser.spi.iobuffer; + exports io.opencmw.serialiser.utils; +} \ No newline at end of file diff --git a/serialiser/src/test/java/io/opencmw/serialiser/IoClassSerialiserSimpleTest.java b/serialiser/src/test/java/io/opencmw/serialiser/IoClassSerialiserSimpleTest.java new file mode 100644 index 00000000..034b3079 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/IoClassSerialiserSimpleTest.java @@ -0,0 +1,42 @@ +package io.opencmw.serialiser; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.utils.TestDataClass; + +/** + * This is a simple example illustrating the use of the IoClassSerialiser. + * + * @author rstein + */ +class IoClassSerialiserSimpleTest { + @Test + void simpleTest() { + final IoBuffer byteBuffer = new FastByteBuffer(10_000); // alt: new ByteBuffer(10_000); + final IoClassSerialiser ioClassSerialiser = new IoClassSerialiser(byteBuffer, BinarySerialiser.class); + // alt: + // ioClassSerialiser.setMatchedIoSerialiser(BinarySerialiser.class); + // ioClassSerialiser.setMatchedIoSerialiser(CmwLightSerialiser.class); + // ioClassSerialiser.setMatchedIoSerialiser(JsonSerialiser.class); + // ioClassSerialiser.setAutoMatchSerialiser(true); // to auto-detect the suitable serialiser based on serialised data header + + TestDataClass data = new TestDataClass(); // object to be serialised + + byteBuffer.reset(); + ioClassSerialiser.serialiseObject(data); // pojo -> serialised data + // [..] stream/write serialised byteBuffer content [..] + // final byte[] serialisedData = byteBuffer.elements(); + // final int serialisedDataSize = byteBuffer.limit(); + + // [..] stream/read serialised byteBuffer content + byteBuffer.flip(); // mark byte-buffer for reading + TestDataClass received = ioClassSerialiser.deserialiseObject(TestDataClass.class); + + // check data equality, etc... + assertEquals(data, received); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/IoClassSerialiserTests.java b/serialiser/src/test/java/io/opencmw/serialiser/IoClassSerialiserTests.java new file mode 100644 index 00000000..82655080 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/IoClassSerialiserTests.java @@ -0,0 +1,399 @@ +package io.opencmw.serialiser; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.parallel.ResourceAccessMode.READ_WRITE; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ByteBuffer; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.JsonSerialiser; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +import de.gsi.dataset.DataSet; +import de.gsi.dataset.spi.DefaultErrorDataSet; +import de.gsi.dataset.spi.utils.MultiArrayBoolean; +import de.gsi.dataset.spi.utils.MultiArrayByte; +import de.gsi.dataset.spi.utils.MultiArrayChar; +import de.gsi.dataset.spi.utils.MultiArrayDouble; +import de.gsi.dataset.spi.utils.MultiArrayFloat; +import de.gsi.dataset.spi.utils.MultiArrayInt; +import de.gsi.dataset.spi.utils.MultiArrayLong; +import de.gsi.dataset.spi.utils.MultiArrayObject; +import de.gsi.dataset.spi.utils.MultiArrayShort; + +/** + * @author rstein + */ +class IoClassSerialiserTests { + private static final int BUFFER_SIZE = 20000; + private static final String GLOBAL_LOCK = "lock"; + + @ParameterizedTest(name = "Serialiser class - {0}") + @ValueSource(classes = { CmwLightSerialiser.class, BinarySerialiser.class, JsonSerialiser.class }) + @ResourceLock(value = GLOBAL_LOCK, mode = READ_WRITE) + void testChangingBuffers(final Class serialiserClass) throws IllegalArgumentException, SecurityException { + final CustomClass2 classUnderTest = new CustomClass2(1.337, 42, "pi equals exactly three!"); + final CustomClass2 classAfterTest = new CustomClass2(); + + IoClassSerialiser serialiser = new IoClassSerialiser(new FastByteBuffer(2 * BUFFER_SIZE), serialiserClass); + serialiser.serialiseObject(classUnderTest); + final byte[] bytes = serialiser.getDataBuffer().elements(); + + final IoClassSerialiser deserialiser = new IoClassSerialiser(new FastByteBuffer(0)); + deserialiser.setDataBuffer(FastByteBuffer.wrap(bytes)); + final Object returnedClass = deserialiser.deserialiseObject(classAfterTest); + + assertSame(classAfterTest, returnedClass); // deserialisation should be in place + assertEquals(classUnderTest, classAfterTest); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + @ResourceLock(value = GLOBAL_LOCK, mode = READ_WRITE) + void testCustomSerialiserIdentity(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + IoClassSerialiser serialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + serialiser.setAutoMatchSerialiser(false); + + final AtomicInteger writerCalled = new AtomicInteger(0); + final AtomicInteger returnCalled = new AtomicInteger(0); + + // add custom serialiser - for more examples see classes in de.gsi.dataset.serializer.spi.iobuffer.*Helper + // provide a writer function + final FieldSerialiser.TriConsumer writeFunction = (io, obj, field) -> { + final Object localObj = field == null || field.getField() == null ? obj : field.getField().get(obj); + if (!(localObj instanceof CustomClass)) { + throw new IllegalArgumentException("object " + obj + " is not of type CustomClass"); + } + CustomClass customClass = (CustomClass) localObj; + // place custom elements/composites etc. here - N.B. ordering is of paramount importance since + // these raw fields are not preceded by field headers + io.getBuffer().putDouble(customClass.testDouble); + io.getBuffer().putInt(customClass.testInt); + io.getBuffer().putString(customClass.testString); + // [..] anything that can be generated with the IoSerialiser and/or IoBuffer interfaces + writerCalled.getAndIncrement(); + }; + + // provide a return function (can usually be re-used for the reader function) + final FieldSerialiser.TriFunction returnFunction = (io, obj, field) -> { + final Object sourceField = field == null ? null : field.getField().get(obj); // get raw class field content + + // place reverse custom elements/composites etc. here - N.B. ordering is of paramount importance since + final double doubleVal = io.getBuffer().getDouble(); + final int intVal = io.getBuffer().getInt(); + final String str = io.getBuffer().getString(); + // generate custom object or modify existing one + returnCalled.getAndIncrement(); + if (sourceField == null) { + return new CustomClass(doubleVal, intVal, str); + } else { + if (!(sourceField instanceof CustomClass)) { + throw new IllegalArgumentException("object " + obj + " is not of type CustomClass"); + } + CustomClass customClass = (CustomClass) sourceField; + customClass.testDouble = doubleVal; + customClass.testInt = intVal; + customClass.testString = str; + return customClass; + } + }; + + serialiser.addClassDefinition(new FieldSerialiser<>( // + /* reader */ (io, obj, field) -> field.getField().set(obj, returnFunction.apply(io, obj, field)), // + /* return */ returnFunction, // + /* write */ writeFunction, CustomClass.class)); + + final CustomClass sourceClass = new CustomClass(1.2, 2, "Hello World!"); + final CustomClass destinationClass = new CustomClass(); + + writerCalled.set(0); + returnCalled.set(0); + // serialise-deserialise DataSet + buffer.reset(); // '0' writing at start of buffer + serialiser.serialiseObject(sourceClass); + buffer.reset(); // reset to read position (==0) + final WireDataFieldDescription root = serialiser.getMatchedIoSerialiser().parseIoStream(true); + root.printFieldStructure(); + buffer.reset(); // reset to read position (==0) + final Object returnObject = serialiser.deserialiseObject(destinationClass); + + assertEquals(sourceClass, returnObject); + // TODO: future upgrade that this tests also passes: + // assertEquals(sourceClass, destinationClass) + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + @SuppressWarnings("unchecked") + void testGenericSerialiserIdentity(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + IoClassSerialiser serialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + + TestClass sourceClass = new TestClass(); + sourceClass.integerBoxed = 1337; + + sourceClass.integerList = new ArrayList<>(); + sourceClass.integerList.add(1); + sourceClass.integerList.add(2); + sourceClass.integerList.add(3); + sourceClass.stringList = new ArrayList<>(); + sourceClass.stringList.add("String1"); + sourceClass.stringList.add("String2"); + sourceClass.dataSet = new DefaultErrorDataSet("test", // + new double[] { 1f, 2f, 3f }, new double[] { 6f, 7f, 8f }, // + new double[] { 0.7f, 0.8f, 0.9f }, new double[] { 7f, 8f, 9f }, 3, false); + sourceClass.dataSetList = new ArrayList<>(); + sourceClass.dataSetList.add(new DefaultErrorDataSet("ListDataSet#1")); + sourceClass.dataSetList.add(new DefaultErrorDataSet("ListDataSet#2")); + sourceClass.dataSetSet = new HashSet<>(); + sourceClass.dataSetSet.add(new DefaultErrorDataSet("SetDataSet#1")); + sourceClass.dataSetSet.add(new DefaultErrorDataSet("SetDataSet#2")); + sourceClass.dataSetQueue = new ArrayDeque<>(); + sourceClass.dataSetQueue.add(new DefaultErrorDataSet("QueueDataSet#1")); + sourceClass.dataSetQueue.add(new DefaultErrorDataSet("QueueDataSet#2")); + + sourceClass.dataSetMap = new HashMap<>(); + sourceClass.dataSetMap.put("dataSet1", new DefaultErrorDataSet("MapDataSet#1")); + sourceClass.dataSetMap.put("dataSet2", new DefaultErrorDataSet("MapDataSet#2")); + + final DefaultErrorDataSet keyDataSet1 = new DefaultErrorDataSet("KeyDataSet#1"); + final DefaultErrorDataSet keyDataSet2 = new DefaultErrorDataSet("KeyDataSet#2"); + sourceClass.dataSetStringMap = new HashMap<>(); + sourceClass.dataSetStringMap.put(keyDataSet1, "keyDataSet1"); + sourceClass.dataSetStringMap.put(keyDataSet2, "keyDataSet2"); + + sourceClass.multiArrayDouble = MultiArrayDouble.wrap(new double[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayFloat = MultiArrayFloat.wrap(new float[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayInt = MultiArrayInt.wrap(new int[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayLong = MultiArrayLong.wrap(new long[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayShort = MultiArrayShort.wrap(new short[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayChar = MultiArrayChar.wrap(new char[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayBoolean = MultiArrayBoolean.wrap(new boolean[] { true, false, false, true, true, false }, new int[] { 2, 3 }); + sourceClass.multiArrayByte = MultiArrayByte.wrap(new byte[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 }); + sourceClass.multiArrayString = MultiArrayObject.wrap(new String[] { "aa", "ba", "ab", "bb", "ac", "bc" }, new int[] { 2, 3 }); // NOPMD NOSONAR -- String type is known + + TestClass destinationClass = new TestClass(); + destinationClass.nullIntegerList = new ArrayList<>(); + destinationClass.nullDataSet = new DefaultErrorDataSet("nullDataSet"); + assertNotEquals(sourceClass.nullIntegerList, destinationClass.nullIntegerList); + assertNotEquals(sourceClass.nullDataSet, destinationClass.nullDataSet); + + // serialise-deserialise DataSet + buffer.reset(); // '0' writing at start of buffer + serialiser.serialiseObject(sourceClass); + buffer.reset(); // reset to read position (==0) + + final WireDataFieldDescription root = serialiser.getMatchedIoSerialiser().parseIoStream(true); + root.printFieldStructure(); + assertEquals(sourceClass.integerBoxed, ((WireDataFieldDescription) root.findChildField("io.opencmw.serialiser.IoClassSerialiserTests$TestClass").findChildField("integerBoxed")).data()); + buffer.reset(); // reset to read position (==0) + serialiser.deserialiseObject(destinationClass); + + buffer.reset(); // reset to read position (==0) + final Object returnedClass = serialiser.deserialiseObject(destinationClass); + + assertSame(returnedClass, destinationClass); // deserialisation should be should be in-place + + assertEquals(sourceClass.integerBoxed, destinationClass.integerBoxed); + + assertEquals(sourceClass.integerList, destinationClass.integerList); + assertEquals(1, destinationClass.integerList.get(0)); + assertEquals(2, destinationClass.integerList.get(1)); + assertEquals(3, destinationClass.integerList.get(2)); + + assertEquals(sourceClass.stringList, destinationClass.stringList); + assertEquals("String1", destinationClass.stringList.get(0)); + assertEquals("String2", destinationClass.stringList.get(1)); + + // assertEquals(sourceClass.emptyIntegerList, destinationClass.emptyIntegerList); cannot assure that null is serialised will map to empty list + // buffer.reset(); // reset to read position (==0) + // final WireDataFieldDescription root = serialiser.getIoSerialiser().parseIoStream(true); + // root.printFieldStructure(); + + assertEquals(sourceClass.dataSet, destinationClass.dataSet); + // assertEquals(sourceClass.nullDataSet, destinationClass.nullDataSet); + + assertEquals(sourceClass.dataSetList, destinationClass.dataSetList); + assertEquals("ListDataSet#1", destinationClass.dataSetList.get(0).getName()); + assertEquals("ListDataSet#2", destinationClass.dataSetList.get(1).getName()); + + assertEquals(sourceClass.dataSetSet, destinationClass.dataSetSet); + assertTrue(destinationClass.dataSetSet.stream().anyMatch(ds -> ds.getName().equals("SetDataSet#1"))); + assertTrue(destinationClass.dataSetSet.stream().anyMatch(ds -> ds.getName().equals("SetDataSet#2"))); + + //assertEquals(sourceClass.dataSetQueue, destinationClass.dataSetQueue); + assertTrue(destinationClass.dataSetQueue.stream().anyMatch(ds -> ds.getName().equals("QueueDataSet#1"))); + assertTrue(destinationClass.dataSetQueue.stream().anyMatch(ds -> ds.getName().equals("QueueDataSet#2"))); + + assertEquals(sourceClass.dataSetMap, destinationClass.dataSetMap); + assertNotNull(destinationClass.dataSetMap.get("dataSet1")); + assertNotNull(destinationClass.dataSetMap.get("dataSet2")); + + assertEquals(sourceClass.dataSetStringMap, destinationClass.dataSetStringMap); + assertEquals("keyDataSet1", destinationClass.dataSetStringMap.get(keyDataSet1)); + assertEquals("keyDataSet2", destinationClass.dataSetStringMap.get(keyDataSet2)); + + assertEquals(sourceClass.multiArrayDouble, destinationClass.multiArrayDouble); + assertArrayEquals(sourceClass.multiArrayDouble.getDimensions(), destinationClass.multiArrayDouble.getDimensions()); + assertArrayEquals(sourceClass.multiArrayDouble.elements(), destinationClass.multiArrayDouble.elements()); + + assertEquals(sourceClass.multiArrayFloat, destinationClass.multiArrayFloat); + assertArrayEquals(sourceClass.multiArrayFloat.getDimensions(), destinationClass.multiArrayFloat.getDimensions()); + assertArrayEquals(sourceClass.multiArrayFloat.elements(), destinationClass.multiArrayFloat.elements()); + + assertEquals(sourceClass.multiArrayInt, destinationClass.multiArrayInt); + assertArrayEquals(sourceClass.multiArrayInt.getDimensions(), destinationClass.multiArrayInt.getDimensions()); + assertArrayEquals(sourceClass.multiArrayInt.elements(), destinationClass.multiArrayInt.elements()); + + assertEquals(sourceClass.multiArrayLong, destinationClass.multiArrayLong); + assertArrayEquals(sourceClass.multiArrayLong.getDimensions(), destinationClass.multiArrayLong.getDimensions()); + assertArrayEquals(sourceClass.multiArrayLong.elements(), destinationClass.multiArrayLong.elements()); + + assertEquals(sourceClass.multiArrayShort, destinationClass.multiArrayShort); + assertArrayEquals(sourceClass.multiArrayShort.getDimensions(), destinationClass.multiArrayShort.getDimensions()); + assertArrayEquals(sourceClass.multiArrayShort.elements(), destinationClass.multiArrayShort.elements()); + + assertEquals(sourceClass.multiArrayByte, destinationClass.multiArrayByte); + assertArrayEquals(sourceClass.multiArrayByte.getDimensions(), destinationClass.multiArrayByte.getDimensions()); + assertArrayEquals(sourceClass.multiArrayByte.elements(), destinationClass.multiArrayByte.elements()); + + assertEquals(sourceClass.multiArrayChar, destinationClass.multiArrayChar); + assertArrayEquals(sourceClass.multiArrayChar.getDimensions(), destinationClass.multiArrayChar.getDimensions()); + assertArrayEquals(sourceClass.multiArrayChar.elements(), destinationClass.multiArrayChar.elements()); + + assertEquals(sourceClass.multiArrayBoolean, destinationClass.multiArrayBoolean); + assertArrayEquals(sourceClass.multiArrayBoolean.getDimensions(), destinationClass.multiArrayBoolean.getDimensions()); + assertArrayEquals(sourceClass.multiArrayBoolean.elements(), destinationClass.multiArrayBoolean.elements()); + + assertEquals(sourceClass.multiArrayString, destinationClass.multiArrayString); + assertArrayEquals(sourceClass.multiArrayString.getDimensions(), destinationClass.multiArrayString.getDimensions()); + assertArrayEquals(sourceClass.multiArrayString.elements(), destinationClass.multiArrayString.elements()); + } + + static class CustomClass { + public double testDouble; + public int testInt; + public String testString; + + public CustomClass() { + this(-1.0, -1, "null string"); // null instantiation + } + public CustomClass(final double testDouble, final int testInt, final String testString) { + this.testDouble = testDouble; + this.testInt = testInt; + this.testString = testString; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final CustomClass that = (CustomClass) o; + return Double.compare(that.testDouble, testDouble) == 0 && testInt == that.testInt && Objects.equals(testString, that.testString); + } + + @Override + public int hashCode() { + return Objects.hash(testDouble, testInt, testString); + } + + @Override + public String toString() { + return "CustomClass(" + testDouble + ", " + testInt + ", " + testString + ")"; + } + } + + /** + * Duplicate of CustomClass, because Custom Class gets registered for a custom (de)serialiser and we don't want that for all tests. + */ + @SuppressWarnings("CanBeFinal") + public static class CustomClass2 { + public double testDouble; + public int testInt; + public String testString; + + public CustomClass2() { + this(-1.0, -1, "null string"); // null instantiation + } + public CustomClass2(final double testDouble, final int testInt, final String testString) { + this.testDouble = testDouble; + this.testInt = testInt; + this.testString = testString; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final CustomClass2 that = (CustomClass2) o; + return Double.compare(that.testDouble, testDouble) == 0 && testInt == that.testInt && Objects.equals(testString, that.testString); + } + + @Override + public int hashCode() { + return Objects.hash(testDouble, testInt, testString); + } + + @Override + public String toString() { + return "CustomClass2(" + testDouble + ", " + testInt + ", " + testString + ")"; + } + } + + /** + * small test class to test (de-)serialisation of wrapped and/or compound object types + */ + static class TestClass { + public Integer integerBoxed; + + public List integerList; + public List stringList; + public List nullIntegerList; + public DataSet dataSet; + public DataSet nullDataSet; + public List dataSetList; + public Set dataSetSet; + public Queue dataSetQueue; + + public Map dataSetMap; + public Map dataSetStringMap; + + public MultiArrayDouble multiArrayDouble; + public MultiArrayFloat multiArrayFloat; + public MultiArrayInt multiArrayInt; + public MultiArrayLong multiArrayLong; + public MultiArrayShort multiArrayShort; + public MultiArrayByte multiArrayByte; + public MultiArrayChar multiArrayChar; + public MultiArrayBoolean multiArrayBoolean; + public MultiArrayObject multiArrayString; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/IoSerialiserTests.java b/serialiser/src/test/java/io/opencmw/serialiser/IoSerialiserTests.java new file mode 100644 index 00000000..6b706fcf --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/IoSerialiserTests.java @@ -0,0 +1,345 @@ +package io.opencmw.serialiser; + +import static org.junit.jupiter.api.Assertions.*; + +import static de.gsi.dataset.DataSet.DIM_X; + +import java.lang.reflect.InvocationTargetException; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.jupiter.params.provider.ValueSource; + +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ByteBuffer; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.JsonSerialiser; +import io.opencmw.serialiser.spi.WireDataFieldDescription; +import io.opencmw.serialiser.spi.helper.MyGenericClass; +import io.opencmw.serialiser.utils.CmwLightHelper; +import io.opencmw.serialiser.utils.JsonHelper; +import io.opencmw.serialiser.utils.SerialiserHelper; +import io.opencmw.serialiser.utils.TestDataClass; + +import de.gsi.dataset.DataSet; +import de.gsi.dataset.spi.DoubleDataSet; + +class IoSerialiserTests { + private static final int BUFFER_SIZE = 50000; + + @ParameterizedTest + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void simpleStreamerTest(final Class bufferClass) throws NoSuchMethodException, SecurityException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(1000000); + + // check reading/writing + final MyGenericClass inputObject = new MyGenericClass(); + MyGenericClass outputObject1 = new MyGenericClass(); + MyGenericClass.setVerboseChecks(true); + + // first test - check for equal initialisation -- this should be trivial + assertEquals(inputObject, outputObject1); + + //final IoBuffer buffer = new FastByteBuffer(1000000); + final IoClassSerialiser serialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + serialiser.serialiseObject(inputObject); + + buffer.flip(); + outputObject1 = serialiser.deserialiseObject(outputObject1); + + // second test - both vectors should have the same initial values + // after serialise/deserialise + assertEquals(inputObject, outputObject1); + + MyGenericClass outputObject2 = new MyGenericClass(); + buffer.reset(); + buffer.clear(); + // modify input object w.r.t. init values + inputObject.modifyValues(); + inputObject.boxedPrimitives.modifyValues(); + inputObject.arrays.modifyValues(); + inputObject.objArrays.modifyValues(); + + serialiser.serialiseObject(inputObject); + + buffer.flip(); + outputObject2 = serialiser.deserialiseObject(outputObject2); + + // third test - both vectors should have the same modified values + assertEquals(inputObject, outputObject2); + } + + @DisplayName("basic custom serialisation/deserialisation identity") + @ParameterizedTest(name = "IoBuffer class - {0} recursion level {1}") + @ArgumentsSource(IoBufferHierarchyArgumentProvider.class) + void testCustomSerialiserIdentity(final Class bufferClass, final int hierarchyLevel) throws NoSuchMethodException, SecurityException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); // TODO: generalise to IoBuffer + + final TestDataClass inputObject = new TestDataClass(10, 100, hierarchyLevel); + final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + + buffer.reset(); + SerialiserHelper.serialiseCustom(ioSerialiser, inputObject); + + buffer.flip(); + SerialiserHelper.deserialiseCustom(ioSerialiser, outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + } + + @ParameterizedTest + @DisplayName("basic DataSet serialisation/deserialisation identity") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testCustomDataSetSerialiserIdentity(final Class bufferClass) throws NoSuchMethodException, SecurityException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + + final IoClassSerialiser serialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + // disable auto serialiser selection because otherwise it will get detected as CMWLightSerialiser (strangely only for FastByteBuffer) + serialiser.setAutoMatchSerialiser(false); + assertEquals(bufferClass, buffer.getClass()); + assertEquals(bufferClass, serialiser.getDataBuffer().getClass()); + + final DoubleDataSet inputObject = new DoubleDataSet("inputObject"); + DataSet outputObject = new DoubleDataSet("outputObject"); + assertNotEquals(inputObject, outputObject); + + buffer.reset(); + serialiser.serialiseObject(inputObject); + buffer.flip(); + outputObject = serialiser.deserialiseObject(outputObject); + + assertEquals(inputObject, outputObject); + + inputObject.add(0.0, 1.0); + inputObject.getAxisDescription(DIM_X).set("time", "s"); + + buffer.reset(); + serialiser.serialiseObject(inputObject); + buffer.flip(); + outputObject = serialiser.deserialiseObject(outputObject); + + assertEquals(inputObject, outputObject); + + inputObject.addDataLabel(0, "MyCustomDataLabel"); + inputObject.addDataStyle(0, "MyCustomDataStyle"); + inputObject.setStyle("myDataSetStyle"); + + buffer.reset(); + serialiser.serialiseObject(inputObject); + buffer.flip(); + outputObject = serialiser.deserialiseObject(outputObject); + + assertEquals(inputObject, outputObject); + } + + @DisplayName("basic POJO serialisation/deserialisation identity") + @ParameterizedTest(name = "IoBuffer class - {0} recursion level {1}") + @ArgumentsSource(IoBufferHierarchyArgumentProvider.class) + void testIoBufferSerialiserIdentity(final Class bufferClass, final int hierarchyLevel) throws IllegalAccessException, InstantiationException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + + final IoClassSerialiser ioClassSerialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + final TestDataClass inputObject = new TestDataClass(10, 100, hierarchyLevel); + final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + + buffer.reset(); + ioClassSerialiser.serialiseObject(inputObject); + + buffer.flip(); + final Object returnedObject = ioClassSerialiser.deserialiseObject(outputObject); + + assertSame(outputObject, returnedObject, "Deserialisation should be in-place"); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + } + + @DisplayName("basic POJO IoSerialiser identity - scan") + @ParameterizedTest(name = "IoSerialiser {0} - IoBuffer class {1} - recursion level {2}") + @ArgumentsSource(IoSerialiserHierarchyArgumentProvider.class) + void testParsingInterface(final Class ioSerialiserClass, final Class bufferClass, final int hierarchyLevel) throws IllegalAccessException, InstantiationException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + assertNotNull(ioSerialiserClass, "ioSerialiserClass being not null"); + assertNotNull(ioSerialiserClass.getConstructor(IoBuffer.class), "Constructor(IoBuffer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + + final IoClassSerialiser ioClassSerialiser = new IoClassSerialiser(buffer, ioSerialiserClass); + assertEquals(ioClassSerialiser.getMatchedIoSerialiser().getClass(), ioSerialiserClass, "matching class type"); + + final TestDataClass inputObject = new TestDataClass(10, 100, hierarchyLevel); + final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + + buffer.reset(); + ioClassSerialiser.serialiseObject(inputObject); + // if (ioSerialiserClass.equals(JsonSerialiser.class)) { + // System.err.println("json output:="); + // final int pos = buffer.position(); + // System.err.println("data = " + new String(buffer.elements(), 0, pos)); + // } + + buffer.flip(); + final Object returnedObject = ioClassSerialiser.deserialiseObject(outputObject); + + assertSame(outputObject, returnedObject, "Deserialisation should be in-place"); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + + buffer.reset(); + final WireDataFieldDescription rootField = ioClassSerialiser.parseWireFormat(); + //rootField.printFieldStructure(); + + assertEquals("ROOT", rootField.getFieldName()); + final WireDataFieldDescription classFields = (WireDataFieldDescription) (rootField.getChildren().get(0)); + for (FieldDescription field : classFields.getChildren()) { + final WireDataFieldDescription wireField = (WireDataFieldDescription) field; + final DataType dataType = wireField.getDataType(); + if ((dataType.isScalar() || dataType.isArray()) && dataType != DataType.START_MARKER && dataType != DataType.END_MARKER) { + final Object data = wireField.data(); + assertNotNull(data, "field non null for: " + wireField.getFieldName()); + assertEquals(dataType, DataType.fromClassType(data.getClass()), "field object type match for: " + wireField.getFieldName()); + } + } + + final WireDataFieldDescription boolField = (WireDataFieldDescription) (classFields.findChildField("bool1")); + assertNotNull(boolField); + assertEquals(inputObject.bool1, boolField.data(), "bool1 data field content"); + assertEquals(inputObject.bool2, ((WireDataFieldDescription) (classFields.findChildField("bool2"))).data(), "bool2 data field content"); + assertEquals(inputObject.byte1, ((WireDataFieldDescription) (classFields.findChildField("byte1"))).data(DataType.BYTE), "byte1 data field content"); + assertEquals(inputObject.byte2, ((WireDataFieldDescription) (classFields.findChildField("byte2"))).data(DataType.BYTE), "byte2 data field content"); + + if (!ioSerialiserClass.equals(JsonSerialiser.class)) { + assertArrayEquals(inputObject.boolArray, (boolean[]) ((WireDataFieldDescription) (classFields.findChildField("boolArray"))).data(), "intArray data field content"); + assertArrayEquals(inputObject.byteArray, (byte[]) ((WireDataFieldDescription) (classFields.findChildField("byteArray"))).data(), "byteArray data field content"); + assertArrayEquals(inputObject.shortArray, (short[]) ((WireDataFieldDescription) (classFields.findChildField("shortArray"))).data(), "shortArray data field content"); + assertArrayEquals(inputObject.intArray, (int[]) ((WireDataFieldDescription) (classFields.findChildField("intArray"))).data(), "intArray data field content"); + assertArrayEquals(inputObject.longArray, (long[]) ((WireDataFieldDescription) (classFields.findChildField("longArray"))).data(), "longArray data field content"); + assertArrayEquals(inputObject.floatArray, (float[]) ((WireDataFieldDescription) (classFields.findChildField("floatArray"))).data(), "floatArray data field content"); + assertArrayEquals(inputObject.doubleArray, (double[]) ((WireDataFieldDescription) (classFields.findChildField("doubleArray"))).data(), "doubleArray data field content"); + assertArrayEquals(inputObject.stringArray, (String[]) ((WireDataFieldDescription) (classFields.findChildField("stringArray"))).data(), "stringArray data field content"); + } + + assertArrayEquals(inputObject.boolArray, (boolean[]) ((WireDataFieldDescription) (classFields.findChildField("boolArray"))).data(DataType.BOOL_ARRAY), "intArray data field content"); + assertArrayEquals(inputObject.byteArray, (byte[]) ((WireDataFieldDescription) (classFields.findChildField("byteArray"))).data(DataType.BYTE_ARRAY), "byteArray data field content"); + assertArrayEquals(inputObject.shortArray, (short[]) ((WireDataFieldDescription) (classFields.findChildField("shortArray"))).data(DataType.SHORT_ARRAY), "shortArray data field content"); + assertArrayEquals(inputObject.intArray, (int[]) ((WireDataFieldDescription) (classFields.findChildField("intArray"))).data(DataType.INT_ARRAY), "intArray data field content"); + assertArrayEquals(inputObject.longArray, (long[]) ((WireDataFieldDescription) (classFields.findChildField("longArray"))).data(DataType.LONG_ARRAY), "longArray data field content"); + assertArrayEquals(inputObject.floatArray, (float[]) ((WireDataFieldDescription) (classFields.findChildField("floatArray"))).data(DataType.FLOAT_ARRAY), "floatArray data field content"); + assertArrayEquals(inputObject.doubleArray, (double[]) ((WireDataFieldDescription) (classFields.findChildField("doubleArray"))).data(DataType.DOUBLE_ARRAY), "doubleArray data field content"); + assertArrayEquals(inputObject.stringArray, (String[]) ((WireDataFieldDescription) (classFields.findChildField("stringArray"))).data(DataType.STRING_ARRAY), "stringArray data field content"); + } + + @DisplayName("benchmark identity tests") + @Test + void benchmarkIdentityTests() { + final TestDataClass inputObject = new TestDataClass(10, 100, 1); + final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + + // execute thrice to ensure that buffer flipping/state is cleaned-up properly + for (int i = 0; i < 2; i++) { + assertDoesNotThrow(() -> {}); + // assertDoesNotThrow(() -> CmwHelper.checkSerialiserIdentity(inputObject, outputObject)); + Assertions.assertDoesNotThrow(() -> CmwLightHelper.checkSerialiserIdentity(inputObject, outputObject)); + assertDoesNotThrow(() -> CmwLightHelper.checkCustomSerialiserIdentity(inputObject, outputObject)); + Assertions.assertDoesNotThrow(() -> JsonHelper.checkSerialiserIdentity(inputObject, outputObject)); + assertDoesNotThrow(() -> JsonHelper.checkCustomSerialiserIdentity(inputObject, outputObject)); + + assertDoesNotThrow(() -> SerialiserHelper.checkSerialiserIdentity(inputObject, outputObject)); + assertDoesNotThrow(() -> SerialiserHelper.checkCustomSerialiserIdentity(inputObject, outputObject)); + // assertDoesNotThrow(() -> FlatBuffersHelper.checkCustomSerialiserIdentity(inputObject, outputObject)); + } + } + + @DisplayName("benchmark performance tests") + @Test + void benchmarkPerformanceTests() { + final TestDataClass inputObject = new TestDataClass(10, 100, 1); + final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + final int nIterations = 1; + // execute thrice to ensure that buffer flipping/state is cleaned-up properly + for (int i = 0; i < 2; i++) { + assertDoesNotThrow(() -> {}); + + // map-only performance + assertDoesNotThrow(() -> JsonHelper.testSerialiserPerformanceMap(nIterations, inputObject)); + // assertDoesNotThrow(() -> CmwHelper.testSerialiserPerformanceMap(nIterations, inputObject, outputObject)); + assertDoesNotThrow(() -> CmwLightHelper.testSerialiserPerformanceMap(nIterations, inputObject)); + assertDoesNotThrow(() -> SerialiserHelper.testSerialiserPerformanceMap(nIterations, inputObject)); + + // custom serialiser performance + assertDoesNotThrow(() -> JsonHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject)); + // assertDoesNotThrow(() -> FlatBuffersHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject)); + assertDoesNotThrow(() -> CmwLightHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject)); + assertDoesNotThrow(() -> SerialiserHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject)); + + // POJO performance + assertDoesNotThrow(() -> JsonHelper.testPerformancePojo(nIterations, inputObject, outputObject)); + assertDoesNotThrow(() -> JsonHelper.testPerformancePojoCodeGen(nIterations, inputObject, outputObject)); + // assertDoesNotThrow(() -> CmwHelper.testPerformancePojo(nIterations, inputObject, outputObject)); + assertDoesNotThrow(() -> CmwLightHelper.testPerformancePojo(nIterations, inputObject, outputObject)); + assertDoesNotThrow(() -> SerialiserHelper.testPerformancePojo(nIterations, inputObject, outputObject)); + } + } + + private static class IoBufferHierarchyArgumentProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext context) { + return Stream.of( + Arguments.of(ByteBuffer.class, 0), + Arguments.of(ByteBuffer.class, 1), + Arguments.of(ByteBuffer.class, 2), + Arguments.of(ByteBuffer.class, 3), + Arguments.of(ByteBuffer.class, 4), + Arguments.of(ByteBuffer.class, 5), + Arguments.of(FastByteBuffer.class, 0), + Arguments.of(FastByteBuffer.class, 1), + Arguments.of(FastByteBuffer.class, 2), + Arguments.of(FastByteBuffer.class, 3), + Arguments.of(FastByteBuffer.class, 4), + Arguments.of(FastByteBuffer.class, 5)); + } + } + + private static class IoSerialiserHierarchyArgumentProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(ExtensionContext context) { + return Stream.of( + Arguments.of(BinarySerialiser.class, ByteBuffer.class, 0), + Arguments.of(BinarySerialiser.class, ByteBuffer.class, 1), + Arguments.of(BinarySerialiser.class, FastByteBuffer.class, 0), + Arguments.of(BinarySerialiser.class, FastByteBuffer.class, 1), + + Arguments.of(CmwLightSerialiser.class, ByteBuffer.class, 0), + Arguments.of(CmwLightSerialiser.class, ByteBuffer.class, 1), + Arguments.of(CmwLightSerialiser.class, FastByteBuffer.class, 0), + Arguments.of(CmwLightSerialiser.class, FastByteBuffer.class, 1), + + Arguments.of(JsonSerialiser.class, ByteBuffer.class, 0), + Arguments.of(JsonSerialiser.class, ByteBuffer.class, 1), + Arguments.of(JsonSerialiser.class, FastByteBuffer.class, 0), + Arguments.of(JsonSerialiser.class, FastByteBuffer.class, 1)); + } + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/annotations/SerialiserAnnotationTests.java b/serialiser/src/test/java/io/opencmw/serialiser/annotations/SerialiserAnnotationTests.java new file mode 100644 index 00000000..3859e6b0 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/annotations/SerialiserAnnotationTests.java @@ -0,0 +1,105 @@ +package io.opencmw.serialiser.annotations; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ByteBuffer; +import io.opencmw.serialiser.spi.ClassFieldDescription; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.WireDataFieldDescription; +import io.opencmw.serialiser.utils.ClassUtils; + +class SerialiserAnnotationTests { + private static final int BUFFER_SIZE = 40000; + + @Test + void testAnnotationGeneration() { + // test annotation parsing on the generation side + final AnnotatedDataClass dataClass = new AnnotatedDataClass(); + final ClassFieldDescription classFieldDescription = ClassUtils.getFieldDescription(dataClass.getClass()); + // classFieldDescription.printFieldStructure(); + + final FieldDescription energyField = classFieldDescription.findChildField("energy".hashCode(), "energy"); + assertNotNull(energyField); + assertEquals("GeV/u", energyField.getFieldUnit()); + assertEquals("energy description", energyField.getFieldDescription()); + assertEquals("OUT", energyField.getFieldDirection()); + assertFalse(energyField.getFieldGroups().isEmpty()); + assertEquals("A", energyField.getFieldGroups().get(0)); + + final FieldDescription temperatureField = classFieldDescription.findChildField("temperature".hashCode(), "temperature"); + assertNotNull(temperatureField); + assertEquals("°C", temperatureField.getFieldUnit()); + assertEquals("important temperature reading", temperatureField.getFieldDescription()); + assertEquals("OUT", temperatureField.getFieldDirection()); + assertFalse(temperatureField.getFieldGroups().isEmpty()); + assertEquals(2, temperatureField.getFieldGroups().size()); + assertEquals("A", temperatureField.getFieldGroups().get(0)); + assertEquals("B", temperatureField.getFieldGroups().get(1)); + } + + @DisplayName("basic custom serialisation/deserialisation identity") + @ParameterizedTest(name = "IoBuffer class - {0} recursion level {1}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testCustomSerialiserIdentity(final Class bufferClass) throws NoSuchMethodException, SecurityException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); + final IoClassSerialiser serialiser = new IoClassSerialiser(buffer, ioSerialiser.getClass()); + + final AnnotatedDataClass inputObject = new AnnotatedDataClass(); + + buffer.reset(); + serialiser.serialiseObject(inputObject); + + buffer.reset(); + final WireDataFieldDescription root = ioSerialiser.parseIoStream(true); + final FieldDescription serialiserFieldDescriptions = root.getChildren().get(0); + + final FieldDescription energyField = serialiserFieldDescriptions.findChildField("energy".hashCode(), "energy"); + assertNotNull(energyField); + assertEquals("GeV/u", energyField.getFieldUnit()); + assertEquals("energy description", energyField.getFieldDescription()); + assertEquals("OUT", energyField.getFieldDirection()); + assertFalse(energyField.getFieldGroups().isEmpty()); + assertEquals("A", energyField.getFieldGroups().get(0)); + + final FieldDescription temperatureField = serialiserFieldDescriptions.findChildField("temperature".hashCode(), "temperature"); + assertNotNull(temperatureField); + assertEquals("°C", temperatureField.getFieldUnit()); + assertEquals("important temperature reading", temperatureField.getFieldDescription()); + assertEquals("OUT", temperatureField.getFieldDirection()); + assertFalse(temperatureField.getFieldGroups().isEmpty()); + assertEquals(2, temperatureField.getFieldGroups().size()); + assertEquals("A", temperatureField.getFieldGroups().get(0)); + assertEquals("B", temperatureField.getFieldGroups().get(1)); + } + + @Description("this class is used to test field annotation") + public static class AnnotatedDataClass { + @MetaInfo(unit = "GeV/u", description = "energy description", direction = "OUT", groups = "A") + public double energy; + + @Unit("°C") + @Description("important temperature reading") + @Direction("OUT") + @Groups({ "A", "B" }) + public double temperature; + + @Unit("V") + @Description("control variable") + @Direction("IN/OUT") + public double controlVariable; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/DataSetSerialiserBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/DataSetSerialiserBenchmark.java new file mode 100644 index 00000000..ea49b772 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/DataSetSerialiserBenchmark.java @@ -0,0 +1,114 @@ +package io.opencmw.serialiser.benchmark; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ByteBuffer; +import io.opencmw.serialiser.spi.FastByteBuffer; + +import de.gsi.dataset.DataSet; +import de.gsi.dataset.spi.DoubleDataSet; +import de.gsi.dataset.testdata.spi.GaussFunction; + +/** + * Simple benchmark to verify that the in-place DataSet (de-)serialiser is not significantly slower than creating a new DataSet + * + * Benchmark Mode Cnt Score Error Units + * DataSetSerialiserBenchmark.serialiserRoundTripByteBufferInplace thrpt 10 5971.023 ± 100.145 ops/s + * DataSetSerialiserBenchmark.serialiserRoundTripByteBufferNewDataSet thrpt 10 5652.462 ± 114.474 ops/s + * DataSetSerialiserBenchmark.serialiserRoundTripFastByteBufferInplace thrpt 10 7468.360 ± 126.494 ops/s + * DataSetSerialiserBenchmark.serialiserRoundTripFastByteBufferNewDataSet thrpt 10 7272.097 ± 170.991 ops/s + * + * @author rstein + */ +@State(Scope.Benchmark) +public class DataSetSerialiserBenchmark { + private static final IoClassSerialiser serialiserFastByteBuffer = new IoClassSerialiser(new FastByteBuffer(200_000), BinarySerialiser.class); + private static final IoClassSerialiser serialiserByteBuffer = new IoClassSerialiser(new ByteBuffer(200_000), BinarySerialiser.class); + private static final DataSet srcDataSet = new DoubleDataSet(new GaussFunction("Gauss-function", 10_000)); + private static final DataSet copyDataSet = new DoubleDataSet(srcDataSet); + private static final TestClass source = new TestClass(); + private static final TestClass copy = new TestClass(); + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void serialiserRoundTripByteBufferInplace(Blackhole blackhole) { + final IoClassSerialiser classSerialiser = serialiserByteBuffer; + source.dataSet = srcDataSet; + copy.dataSet = copyDataSet; + + // serialise-deserialise DataSet + classSerialiser.getDataBuffer().reset(); // '0' writing at start of buffer + classSerialiser.serialiseObject(source); + + classSerialiser.getDataBuffer().flip(); // reset to read position (==0) + classSerialiser.deserialiseObject(copy); + + blackhole.consume(copy); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void serialiserRoundTripByteBufferNewDataSet(Blackhole blackhole) { + final IoClassSerialiser classSerialiser = serialiserByteBuffer; + source.dataSet = srcDataSet; + copy.dataSet = null; + + // serialise-deserialise DataSet + classSerialiser.getDataBuffer().reset(); // '0' writing at start of buffer + classSerialiser.serialiseObject(source); + + classSerialiser.getDataBuffer().flip(); // reset to read position (==0) + classSerialiser.deserialiseObject(copy); + + blackhole.consume(copy); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void serialiserRoundTripFastByteBufferInplace(Blackhole blackhole) { + final IoClassSerialiser classSerialiser = serialiserFastByteBuffer; + source.dataSet = srcDataSet; + copy.dataSet = copyDataSet; + + // serialise-deserialise DataSet + classSerialiser.getDataBuffer().reset(); // '0' writing at start of buffer + classSerialiser.serialiseObject(source); + + classSerialiser.getDataBuffer().flip(); // reset to read position (==0) + classSerialiser.deserialiseObject(copy); + + blackhole.consume(copy); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void serialiserRoundTripFastByteBufferNewDataSet(Blackhole blackhole) { + final IoClassSerialiser classSerialiser = serialiserFastByteBuffer; + source.dataSet = srcDataSet; + copy.dataSet = null; + + // serialise-deserialise DataSet + classSerialiser.getDataBuffer().reset(); // '0' writing at start of buffer + classSerialiser.serialiseObject(source); + + classSerialiser.getDataBuffer().flip(); // reset to read position (==0) + classSerialiser.deserialiseObject(copy); + + blackhole.consume(copy); + } + + static class TestClass { + public DataSet dataSet; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/FastByteBufferBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/FastByteBufferBenchmark.java new file mode 100644 index 00000000..70c06c08 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/FastByteBufferBenchmark.java @@ -0,0 +1,37 @@ +package io.opencmw.serialiser.benchmark; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import io.opencmw.serialiser.spi.FastByteBuffer; + +/** + * Simple benchmark to evaluate the effect of range checks on FastByteBuffer performance + * + * Without range check: + * FastByteBufferBenchmark.putInts thrpt 10 40419,996 ± 3480,205 ops/s + * With range check: + * FastByteBufferBenchmark.putInts thrpt 10 40108,335 ± 912,071 ops/s + * + * @author akrimm + */ +@State(Scope.Benchmark) +public class FastByteBufferBenchmark { + private static final FastByteBuffer fastByteBuffer = new FastByteBuffer(200_000); + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void putInts(Blackhole blackhole) { + fastByteBuffer.reset(); + for (int i = 0; i < 200_000 / 16; ++i) { + fastByteBuffer.putInt(blackhole.i1); + fastByteBuffer.putInt(blackhole.i2); + } + blackhole.consume(fastByteBuffer.elements()); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/JsonSelectionBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/JsonSelectionBenchmark.java new file mode 100644 index 00000000..38351a79 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/JsonSelectionBenchmark.java @@ -0,0 +1,128 @@ +package io.opencmw.serialiser.benchmark; + +import java.io.IOException; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import io.opencmw.serialiser.utils.TestDataClass; + +import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.jsoniter.JsonIterator; +import com.jsoniter.output.EncodingMode; +import com.jsoniter.output.JsonStream; +import com.jsoniter.spi.DecodingMode; + +/** + * simple benchmark to evaluate various JSON libraries. + * N.B. This is not intended as a complete JSON serialiser evaluation but to indicate some rough trends. + * + * testClassId 1: being a string-heavy test data class + * testClassId 2: being a numeric-data-heavy test data class + * + * Benchmark (testClassId) Mode Cnt Score Error Units + * JsonSelectionBenchmark.pojoFastJson string-heavy thrpt 10 12857.850 ± 109.050 ops/s + * JsonSelectionBenchmark.pojoFastJson numeric-heavy thrpt 10 91.458 ± 0.437 ops/s + * JsonSelectionBenchmark.pojoGson string-heavy thrpt 10 6253.698 ± 50.267 ops/s + * JsonSelectionBenchmark.pojoGson numeric-heavy thrpt 10 48.215 ± 0.265 ops/s + * JsonSelectionBenchmark.pojoJackson string-heavy thrpt 10 16563.604 ± 244.329 ops/s + * JsonSelectionBenchmark.pojoJackson numeric-heavy thrpt 10 135.780 ± 1.074 ops/s + * JsonSelectionBenchmark.pojoJsonIter string-heavy thrpt 10 10733.539 ± 35.605 ops/s + * JsonSelectionBenchmark.pojoJsonIter numeric-heavy thrpt 10 86.629 ± 1.122 ops/s + * JsonSelectionBenchmark.pojoJsonIterCodeGen string-heavy thrpt 10 41048.034 ± 396.628 ops/s + * JsonSelectionBenchmark.pojoJsonIterCodeGen numeric-heavy thrpt 10 377.412 ± 9.755 ops/s + * + * Process finished with exit code 0 + */ +@State(Scope.Benchmark) +public class JsonSelectionBenchmark { + private static final String INPUT_OBJECT_NAME_1 = "string-heavy"; + private static final String INPUT_OBJECT_NAME_2 = "numeric-heavy"; + private static final TestDataClass inputObject1 = new TestDataClass(10, 100, 1); // string-heavy + private static final TestDataClass inputObject2 = new TestDataClass(10000, 0, 0); // numeric-heavy + private static final GsonBuilder builder = new GsonBuilder(); + private static final Gson gson = builder.create(); + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final boolean TEST_IDENTITY = true; + @Param({ INPUT_OBJECT_NAME_1, INPUT_OBJECT_NAME_2 }) + private String testClassId; + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoFastJson(Blackhole blackhole) { + final String serialisedData = JSON.toJSONString(getTestClass(testClassId)); // from object to JSON String + final TestDataClass outputPojo = JSON.parseObject(serialisedData, TestDataClass.class); // from JSON String to object + assert !TEST_IDENTITY || getTestClass(testClassId).equals(outputPojo); + blackhole.consume(serialisedData); + blackhole.consume(outputPojo); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoGson(Blackhole blackhole) { + final String serialisedData = gson.toJson(getTestClass(testClassId)); // from object to JSON String + final TestDataClass outputPojo = gson.fromJson(serialisedData, TestDataClass.class); // from JSON String to object + assert !TEST_IDENTITY || getTestClass(testClassId).equals(outputPojo); + blackhole.consume(serialisedData); + blackhole.consume(outputPojo); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoJackson(Blackhole blackhole) { + try { + // set this since the other libraries also (de-)serialise private fields (N.B. TestDataClass fields are all public) + objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + final String serialisedData = objectMapper.writeValueAsString(getTestClass(testClassId)); // from object to JSON String + final TestDataClass outputPojo = objectMapper.readValue(serialisedData, TestDataClass.class); // from JSON String to object + assert !TEST_IDENTITY || getTestClass(testClassId).equals(outputPojo); + blackhole.consume(serialisedData); + blackhole.consume(outputPojo); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoJsonIter(Blackhole blackhole) { + JsonStream.setMode(EncodingMode.REFLECTION_MODE); + JsonIterator.setMode(DecodingMode.REFLECTION_MODE); + final String serialisedData = JsonStream.serialize(getTestClass(testClassId)); // from object to JSON String + final TestDataClass outputPojo = JsonIterator.deserialize(serialisedData, TestDataClass.class); // from JSON String to object + assert !TEST_IDENTITY || getTestClass(testClassId).equals(outputPojo); + blackhole.consume(serialisedData); + blackhole.consume(outputPojo); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoJsonIterCodeGen(Blackhole blackhole) { + JsonStream.setMode(EncodingMode.DYNAMIC_MODE); + JsonIterator.setMode(DecodingMode.DYNAMIC_MODE_AND_MATCH_FIELD_WITH_HASH); + final String serialisedData = JsonStream.serialize(getTestClass(testClassId)); // from object to JSON String + final TestDataClass outputPojo = JsonIterator.deserialize(serialisedData, TestDataClass.class); // from JSON String to object + assert !TEST_IDENTITY || getTestClass(testClassId).equals(outputPojo); + blackhole.consume(serialisedData); + blackhole.consume(outputPojo); + } + + private static TestDataClass getTestClass(final String arg) { + return INPUT_OBJECT_NAME_1.equals(arg) ? inputObject1 : inputObject2; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/ReflectionBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/ReflectionBenchmark.java new file mode 100644 index 00000000..b37286a1 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/ReflectionBenchmark.java @@ -0,0 +1,192 @@ +package io.opencmw.serialiser.benchmark; + +import java.lang.reflect.Field; +import java.util.Objects; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import sun.misc.Unsafe; + +/** + * Benchmark to compare, test and rationalise some assumptions that went into the serialiser refactoring + * + * last test output (openjdk 11.0.7 2020-04-14, took 24 min): + * Benchmark Mode Cnt Score Error Units + * ReflectionBenchmark.fieldAccess1ViaMethod thrpt 10 368156046.779 ± 29954108.137 ops/s + * ReflectionBenchmark.fieldAccess2ViaField thrpt 10 160816173.982 ± 16004563.682 ops/s + * ReflectionBenchmark.fieldAccess3ViaFieldSetDouble thrpt 10 151854907.619 ± 10826489.476 ops/s + * ReflectionBenchmark.fieldAccess4ViaOptimisedField thrpt 10 195051859.819 ± 8803389.807 ops/s + * ReflectionBenchmark.fieldAccess5ViaOptimisedFieldSetDouble thrpt 10 201686198.694 ± 10422965.353 ops/s + * ReflectionBenchmark.fieldAccess6ViaDirectMemoryAccess thrpt 10 341641937.437 ± 53603148.397 ops/s + * + * @author rstein + */ +@State(Scope.Benchmark) +public class ReflectionBenchmark { + public static final int LOOP_COUNT = 100_000_000; + // static final Unsafe unsafe = jdk.internal.misc.Unsafe.getUnsafe(); + private static final Unsafe unsafe; // NOPMD + static { + // get an instance of the otherwise private 'Unsafe' class + try { + final Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (Unsafe) field.get(null); + } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + throw new SecurityException(e); // NOPMD + } + } + private final Field field = getField(); + private final Field fieldOptimised = getOptimisedField(); + private final long fieldOffset = getFieldOffset(); + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fieldAccess1ViaMethod(Blackhole blackhole, final MyData data) { + data.setValue(data.a); + blackhole.consume(data); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fieldAccess2ViaField(Blackhole blackhole, final MyData data) { + try { + Objects.requireNonNull(field, "field is null").set(data, data.a); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + blackhole.consume(data); + } + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fieldAccess3ViaFieldSetDouble(Blackhole blackhole, final MyData data) { + try { + Objects.requireNonNull(field, "field is null").setDouble(data, data.a); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + blackhole.consume(data); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fieldAccess4ViaOptimisedField(Blackhole blackhole, final MyData data) { + try { + Objects.requireNonNull(fieldOptimised, "field is null").set(data, data.a); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + blackhole.consume(data); + } + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fieldAccess5ViaOptimisedFieldSetDouble(Blackhole blackhole, final MyData data) { + try { + Objects.requireNonNull(fieldOptimised, "field is null").setDouble(data, data.a); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + blackhole.consume(data); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fieldAccess6ViaDirectMemoryAccess(Blackhole blackhole, final MyData data) { + unsafe.putDouble(data, fieldOffset, data.a); + blackhole.consume(data); + } + + public static void main(String[] args) throws Throwable { + MyData data = new MyData(); + + long start = System.currentTimeMillis(); + for (int i = 0; i < LOOP_COUNT; ++i) { + data.setValue(i); + } + System.err.println("access via method: " + (System.currentTimeMillis() - start)); + + Field field = MyData.class.getDeclaredField("value"); + start = System.currentTimeMillis(); + for (int i = 0; i < LOOP_COUNT; ++i) { + field.set(data, (double) i); + } + System.err.println("access via reflection: " + (System.currentTimeMillis() - start)); + + field.setAccessible(true); // Optimization + start = System.currentTimeMillis(); + for (int i = 0; i < LOOP_COUNT; ++i) { + field.set(data, (double) i); + } + System.err.println("access via opt.-refl.: " + (System.currentTimeMillis() - start)); + + start = System.currentTimeMillis(); + for (int i = 0; i < LOOP_COUNT; ++i) { + field.setDouble(data, i); + } + System.err.println("access via setDouble: " + (System.currentTimeMillis() - start)); + + final long fieldOffset = unsafe.objectFieldOffset(field); + start = System.currentTimeMillis(); + for (int i = 0; i < LOOP_COUNT; ++i) { + unsafe.putDouble(data, fieldOffset, i); + } + System.err.println("access via unsafe: " + (System.currentTimeMillis() - start)); + } + + private static Field getField() { + try { + return MyData.class.getDeclaredField("value"); + } catch (NoSuchFieldException e) { + e.printStackTrace(); + } + return null; + } + + private static long getFieldOffset() { + try { + final Field field = MyData.class.getDeclaredField("value"); + field.setAccessible(true); + return unsafe.objectFieldOffset(field); + } catch (NoSuchFieldException e) { + e.printStackTrace(); + } + return -1; + } + + private static Field getOptimisedField() { + try { + final Field field = MyData.class.getDeclaredField("value"); + field.setAccessible(true); + return field; + } catch (NoSuchFieldException e) { + e.printStackTrace(); + } + return null; + } + + @State(Scope.Thread) + @SuppressWarnings("CanBeFinal") + public static class MyData { + public double a = 5.0; + public double value; + + public double getValue() { + return value; + } + + public void setValue(double value) { + this.value = value; + } + } +} \ No newline at end of file diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserAssumptionsBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserAssumptionsBenchmark.java new file mode 100644 index 00000000..bf97e2aa --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserAssumptionsBenchmark.java @@ -0,0 +1,271 @@ +package io.opencmw.serialiser.benchmark; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.charset.StandardCharsets; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import sun.misc.Unsafe; + +/** + * Benchmark to compare, test and rationalise some assumptions that went into the serialiser refactoring + * + * last test output (openjdk 11.0.7 2020-04-14, took ~1:15h): + Benchmark Mode Cnt Score Error Units + SerialiserAssumptionsBenchmark.fluentDesignVoid thrpt 10 471049302.874 ± 38950975.384 ops/s + SerialiserAssumptionsBenchmark.fluentDesignWithReturn thrpt 10 254339145.447 ± 5897376.654 ops/s + SerialiserAssumptionsBenchmark.fluentDesignWithoutReturn thrpt 10 476501604.954 ± 40973207.961 ops/s + SerialiserAssumptionsBenchmark.functionWithArray thrpt 10 221467425.933 ± 3002743.890 ops/s + SerialiserAssumptionsBenchmark.functionWithSingleArgument thrpt 10 287031569.929 ± 5614312.539 ops/s + SerialiserAssumptionsBenchmark.functionWithVarargsArrayArgument thrpt 10 218877961.349 ± 4692333.825 ops/s + SerialiserAssumptionsBenchmark.functionWithVarargsMultiArguments thrpt 10 142480433.294 ± 13146238.914 ops/s + SerialiserAssumptionsBenchmark.functionWithVarargsSingleArgument thrpt 10 165330552.790 ± 17698360.163 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithCharsetASCII thrpt 10 40173625.939 ± 735728.322 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithCharsetISO8859 thrpt 10 54217550.808 ± 1318340.645 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithCharsetUTF8 thrpt 10 41330615.597 ± 632412.774 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithCharsetUTF8_UTF8 thrpt 10 13048847.527 ± 66585.989 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithOutCharsetASCII thrpt 10 49542211.521 ± 2229389.418 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithOutCharsetISO8859 thrpt 10 49123986.088 ± 2657664.797 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithOutCharsetISO8859_V2 thrpt 10 100884194.645 ± 9426023.968 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithOutCharsetISO8859_V3 thrpt 10 71306482.793 ± 2909269.756 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithOutCharsetUTF8 thrpt 10 49503339.953 ± 2376939.023 ops/s + SerialiserAssumptionsBenchmark.stringAllocationWithOutCharsetUTF8_UTF8 thrpt 10 13860025.901 ± 244519.585 ops/s + * + * @author rstein + */ +@State(Scope.Benchmark) +public class SerialiserAssumptionsBenchmark { + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fluentDesignVoid(Blackhole blackhole, final MyData data) { + func1(blackhole, data.a); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public SerialiserAssumptionsBenchmark fluentDesignWithReturn(Blackhole blackhole, final MyData data) { + return func2(blackhole, data.a); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void fluentDesignWithoutReturn(Blackhole blackhole, final MyData data) { + func2(blackhole, data.a); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public int[] functionWithArray(Blackhole blackhole, final MyData data) { + return f2(blackhole, data.dim); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public int functionWithSingleArgument(Blackhole blackhole, final MyData data) { + return f3(blackhole, data.a); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public int[] functionWithVarargsArrayArgument(Blackhole blackhole, final MyData data) { + return f1(blackhole, data.dim); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public int[] functionWithVarargsMultiArguments(Blackhole blackhole, final MyData data) { + return f1(blackhole, data.a, data.b, data.c); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public int[] functionWithVarargsSingleArgument(Blackhole blackhole, final MyData data) { + return f1(blackhole, data.a); + } + + @Setup() + public void initialize() { + // add variables to initialise here + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithCharsetASCII(final MyData data) { + return new String(data.byteASCII, 0, data.byteASCII.length, StandardCharsets.US_ASCII); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithCharsetISO8859(final MyData data) { + return new String(data.byteISO8859, 0, data.byteISO8859.length, StandardCharsets.ISO_8859_1); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithCharsetUTF8(final MyData data) { + return new String(data.byteISO8859, 0, data.byteISO8859.length, StandardCharsets.UTF_8); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithCharsetUTF8_UTF8(final MyData data) { + return new String(data.byteUTF8, 0, data.byteUTF8.length, StandardCharsets.UTF_8); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithOutCharsetASCII(final MyData data) { + return new String(data.byteASCII, 0, data.byteASCII.length); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithOutCharsetISO8859(final MyData data) { + return new String(data.byteISO8859, 0, data.byteISO8859.length); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithOutCharsetISO8859_V2(final MyData data) { + return new String(data.byteISO8859, 0, 0, data.byteISO8859.length); // NOPMD NOSONAR fast implementation + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithOutCharsetISO8859_V3(final MyData data) { + return FastStringBuilder.iso8859BytesToString(data.byteISO8859, 0, data.byteISO8859.length); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithOutCharsetUTF8(final MyData data) { + return new String(data.byteISO8859, 0, data.byteISO8859.length); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public String stringAllocationWithOutCharsetUTF8_UTF8(final MyData data) { + return new String(data.byteUTF8, 0, data.byteUTF8.length); + } + + private int[] f1(Blackhole blackhole, int... array) { + blackhole.consume(array); + return array; + } + + private int[] f2(Blackhole blackhole, int[] array) { + blackhole.consume(array); + return array; + } + + private int f3(Blackhole blackhole, int val) { + blackhole.consume(val); + return val; + } + + private void func1(Blackhole blackhole, final double val) { + blackhole.consume(val); + } + + private SerialiserAssumptionsBenchmark func2(Blackhole blackhole, final double val) { + blackhole.consume(val); + return this; + } + + @State(Scope.Thread) + public static class MyData { + private static final int ARRAY_SIZE = 10; + public int a = 1; + public int b = 2; + public int c = 2; + public int[] dim = { a, b, c }; + public String stringASCII = "Hello World!"; + public String stringISO8859 = "Hello World!"; + public String stringUTF8 = "Γειά σου Κόσμε!"; + public byte[] byteASCII = stringASCII.getBytes(StandardCharsets.US_ASCII); + public byte[] byteISO8859 = stringISO8859.getBytes(StandardCharsets.ISO_8859_1); + public byte[] byteUTF8 = stringUTF8.getBytes(StandardCharsets.UTF_8); + public String[] arrayISO8859 = new String[ARRAY_SIZE]; + public String[] arrayUTF8 = new String[ARRAY_SIZE]; + + public MyData() { + for (int i = 0; i < ARRAY_SIZE; i++) { + arrayISO8859[i] = stringISO8859; + arrayUTF8[i] = stringUTF8; + } + } + } + + /** + * Simple helper class to generate (a little bit) faster Strings from byte arrays ;-) + * N.B. bypassing some of the redundant (null-pointer, byte array size, etc.) safety checks gains up to about 80% performance. + */ + @SuppressWarnings("PMD") + public static class FastStringBuilder { + private static final Logger LOGGER = LoggerFactory.getLogger(FastStringBuilder.class); + private static final Field fieldValue; + private static final long FIELD_VALUE_OFFSET; + private static final Unsafe unsafe; // NOPMD + static { + try { + final Field field = Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + unsafe = (Unsafe) field.get(null); + } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + throw new SecurityException(e); // NOPMD + } + + Field tempVal = null; + long offset = 0; + try { + tempVal = String.class.getDeclaredField("value"); + tempVal.setAccessible(true); + + final Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(tempVal, tempVal.getModifiers() & ~Modifier.FINAL); + offset = unsafe.objectFieldOffset(tempVal); + } catch (NoSuchFieldException | IllegalAccessException e) { + LOGGER.atError().setCause(e).log("could not initialise String field references"); + } finally { + fieldValue = tempVal; + FIELD_VALUE_OFFSET = offset; + } + } + + public static String iso8859BytesToString(final byte[] ba, final int offset, final int length) { + final String retVal = ""; // NOPMD - on purpose allocating new object + final byte[] array = new byte[length]; + System.arraycopy(ba, offset, array, 0, length); + unsafe.putObject(retVal, FIELD_VALUE_OFFSET, array); + return retVal; + } + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserBenchmark.java new file mode 100644 index 00000000..b6975753 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserBenchmark.java @@ -0,0 +1,196 @@ +package io.opencmw.serialiser.benchmark; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import io.opencmw.serialiser.utils.CmwLightHelper; +import io.opencmw.serialiser.utils.FlatBuffersHelper; +import io.opencmw.serialiser.utils.JsonHelper; +import io.opencmw.serialiser.utils.SerialiserHelper; +import io.opencmw.serialiser.utils.TestDataClass; + +import com.jsoniter.JsonIterator; +import com.jsoniter.output.EncodingMode; +import com.jsoniter.output.JsonStream; +import com.jsoniter.spi.DecodingMode; + +/** + * More thorough (JMH-based)) benchmark of various internal and external serialiser protocols. + * Test consists of a simple repeated POJO->serialised->byte[] buffer -> de-serialisation -> POJO + comparison checks. + * N.B. this isn't as precise as the JMH tests but gives a rough idea whether the protocol degraded or needs to be improved. + * + * Benchmark (testClassId) Mode Cnt Score Error Units + * SerialiserBenchmark.customCmwLight string-heavy thrpt 10 49954.479 ± 560.726 ops/s + * SerialiserBenchmark.customCmwLight numeric-heavy thrpt 10 22433.828 ± 195.939 ops/s + * SerialiserBenchmark.customFlatBuffer string-heavy thrpt 10 18446.085 ± 71.311 ops/s + * SerialiserBenchmark.customFlatBuffer numeric-heavy thrpt 10 233.869 ± 7.314 ops/s + * SerialiserBenchmark.customIoSerialiser string-heavy thrpt 10 53638.035 ± 367.122 ops/s + * SerialiserBenchmark.customIoSerialiser numeric-heavy thrpt 10 24277.732 ± 200.380 ops/s + * SerialiserBenchmark.customIoSerialiserOptim string-heavy thrpt 10 79759.984 ± 799.944 ops/s + * SerialiserBenchmark.customIoSerialiserOptim numeric-heavy thrpt 10 24192.169 ± 419.019 ops/s + * SerialiserBenchmark.customJson string-heavy thrpt 10 17619.026 ± 250.917 ops/s + * SerialiserBenchmark.customJson numeric-heavy thrpt 10 138.461 ± 2.972 ops/s + * SerialiserBenchmark.mapCmwLight string-heavy thrpt 10 79273.547 ± 2487.931 ops/s + * SerialiserBenchmark.mapCmwLight numeric-heavy thrpt 10 67374.131 ± 954.149 ops/s + * SerialiserBenchmark.mapIoSerialiser string-heavy thrpt 10 81295.197 ± 2391.616 ops/s + * SerialiserBenchmark.mapIoSerialiser numeric-heavy thrpt 10 67701.564 ± 1062.641 ops/s + * SerialiserBenchmark.mapIoSerialiserOptimized string-heavy thrpt 10 115008.285 ± 2390.426 ops/s + * SerialiserBenchmark.mapIoSerialiserOptimized numeric-heavy thrpt 10 68879.735 ± 1403.197 ops/s + * SerialiserBenchmark.mapJson string-heavy thrpt 10 14474.142 ± 1227.165 ops/s + * SerialiserBenchmark.mapJson numeric-heavy thrpt 10 163.928 ± 0.968 ops/s + * SerialiserBenchmark.pojoCmwLight string-heavy thrpt 10 41821.232 ± 217.594 ops/s + * SerialiserBenchmark.pojoCmwLight numeric-heavy thrpt 10 33820.451 ± 568.264 ops/s + * SerialiserBenchmark.pojoIoSerialiser string-heavy thrpt 10 41899.128 ± 940.030 ops/s + * SerialiserBenchmark.pojoIoSerialiser numeric-heavy thrpt 10 33918.815 ± 376.551 ops/s + * SerialiserBenchmark.pojoIoSerialiserOptim string-heavy thrpt 10 53811.486 ± 920.474 ops/s + * SerialiserBenchmark.pojoIoSerialiserOptim numeric-heavy thrpt 10 32463.267 ± 635.326 ops/s + * SerialiserBenchmark.pojoJson string-heavy thrpt 10 23327.701 ± 288.871 ops/s + * SerialiserBenchmark.pojoJson numeric-heavy thrpt 10 161.396 ± 3.040 ops/s + * SerialiserBenchmark.pojoJsonCodeGen string-heavy thrpt 10 23586.818 ± 470.233 ops/s + * SerialiserBenchmark.pojoJsonCodeGen numeric-heavy thrpt 10 163.250 ± 1.254 ops/s + * + * @author rstein + */ +@State(Scope.Benchmark) +public class SerialiserBenchmark { + private static final String INPUT_OBJECT_NAME_1 = "string-heavy"; + private static final String INPUT_OBJECT_NAME_2 = "numeric-heavy"; + private static final TestDataClass inputObject1 = new TestDataClass(10, 100, 1); // string-heavy + private static final TestDataClass inputObject2 = new TestDataClass(10000, 0, 0); // numeric-heavy + private static final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + @Param({ INPUT_OBJECT_NAME_1, INPUT_OBJECT_NAME_2 }) + private String testClassId; + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void mapJson(Blackhole blackhole) { + JsonHelper.testSerialiserPerformanceMap(1, getTestClass(testClassId)); + blackhole.consume(getTestClass(testClassId)); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void mapCmwLight(Blackhole blackhole) { + CmwLightHelper.testSerialiserPerformanceMap(1, getTestClass(testClassId)); + blackhole.consume(getTestClass(testClassId)); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void mapIoSerialiser(Blackhole blackhole) { + SerialiserHelper.testSerialiserPerformanceMap(1, getTestClass(testClassId)); + blackhole.consume(getTestClass(testClassId)); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void mapIoSerialiserOptimized(Blackhole blackhole) { + SerialiserHelper.getBinarySerialiser().setEnforceSimpleStringEncoding(true); + SerialiserHelper.getBinarySerialiser().setPutFieldMetaData(false); + SerialiserHelper.testSerialiserPerformanceMap(1, getTestClass(testClassId)); + blackhole.consume(getTestClass(testClassId)); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void customJson(Blackhole blackhole) { + JsonHelper.testCustomSerialiserPerformance(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void customFlatBuffer(Blackhole blackhole) { + // N.B. internally FlatBuffer's FlexBuffer API is being used + // rationale: needed to compare libraries that allow loose coupling between server/client-side domain object definition + FlatBuffersHelper.testCustomSerialiserPerformance(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void customCmwLight(Blackhole blackhole) { + CmwLightHelper.testCustomSerialiserPerformance(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void customIoSerialiser(Blackhole blackhole) { + SerialiserHelper.testCustomSerialiserPerformance(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void customIoSerialiserOptim(Blackhole blackhole) { + SerialiserHelper.getBinarySerialiser().setEnforceSimpleStringEncoding(true); + SerialiserHelper.getBinarySerialiser().setPutFieldMetaData(false); + SerialiserHelper.testCustomSerialiserPerformance(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoJson(Blackhole blackhole) { + JsonStream.setMode(EncodingMode.REFLECTION_MODE); + JsonIterator.setMode(DecodingMode.REFLECTION_MODE); + JsonHelper.testPerformancePojoCodeGen(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoJsonCodeGen(Blackhole blackhole) { + JsonStream.setMode(EncodingMode.DYNAMIC_MODE); + JsonIterator.setMode(DecodingMode.DYNAMIC_MODE_AND_MATCH_FIELD_WITH_HASH); + JsonHelper.testPerformancePojoCodeGen(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoCmwLight(Blackhole blackhole) { + CmwLightHelper.testPerformancePojo(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoIoSerialiser(Blackhole blackhole) { + SerialiserHelper.testPerformancePojo(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + @Benchmark + @Warmup(iterations = 1) + @Fork(value = 2, warmups = 2) + public void pojoIoSerialiserOptim(Blackhole blackhole) { + SerialiserHelper.getBinarySerialiser().setEnforceSimpleStringEncoding(true); + SerialiserHelper.getBinarySerialiser().setPutFieldMetaData(false); + SerialiserHelper.testPerformancePojo(1, getTestClass(testClassId), outputObject); + blackhole.consume(outputObject); + } + + private static TestDataClass getTestClass(final String arg) { + return INPUT_OBJECT_NAME_1.equals(arg) ? inputObject1 : inputObject2; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserQuickBenchmark.java b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserQuickBenchmark.java new file mode 100644 index 00000000..fd1f20a8 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/benchmark/SerialiserQuickBenchmark.java @@ -0,0 +1,113 @@ +package io.opencmw.serialiser.benchmark; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.utils.CmwLightHelper; +import io.opencmw.serialiser.utils.FlatBuffersHelper; +import io.opencmw.serialiser.utils.JsonHelper; +import io.opencmw.serialiser.utils.SerialiserHelper; +import io.opencmw.serialiser.utils.TestDataClass; + +/** + * Simple (rough) benchmark of various internal and external serialiser protocols. + * Test consists of a simple repeated POJO->serialised->byte[] buffer -> de-serialisation -> POJO + comparison checks. + * N.B. this isn't as precise as the JMH tests but gives a rough idea whether the protocol degraded or needs to be improved. + * + * Example output - numbers should be compared relatively (nIterations = 100000): + * (openjdk 11.0.7 2020-04-14, ASCII-only, nSizePrimitiveArrays = 10, nSizeString = 100, nestedClassRecursion = 1) + * [..] more string-heavy TestDataClass + * - run 1 + * - JSON Serializer (Map only) throughput = 371.4 MB/s for 5.2 kB per test run (took 1413.0 ms) + * - CMW Serializer (Map only) throughput = 220.2 MB/s for 6.3 kB per test run (took 2871.0 ms) + * - CmwLight Serializer (Map only) throughput = 683.1 MB/s for 6.4 kB per test run (took 935.0 ms) + * - IO Serializer (Map only) throughput = 810.0 MB/s for 7.4 kB per test run (took 908.0 ms) + * + * - FlatBuffers (custom FlexBuffers) throughput = 173.7 MB/s for 6.1 kB per test run (took 3536.0 ms) + * - CmwLight Serializer (custom) throughput = 460.5 MB/s for 6.4 kB per test run (took 1387.0 ms) + * - IO Serializer (custom) throughput = 545.0 MB/s for 7.3 kB per test run (took 1344.0 ms) + * + * - JSON Serializer (POJO) throughput = 53.8 MB/s for 5.2 kB per test run (took 9747.0 ms) + * - CMW Serializer (POJO) throughput = 182.8 MB/s for 6.3 kB per test run (took 3458.0 ms) + * - CmwLight Serializer (POJO) throughput = 329.2 MB/s for 6.3 kB per test run (took 1906.0 ms) + * - IO Serializer (POJO) throughput = 374.9 MB/s for 7.2 kB per test run (took 1925.0 ms) + * + * [..] more primitive-array-heavy TestDataClass + * (openjdk 11.0.7 2020-04-14, UTF8, nSizePrimitiveArrays = 1000, nSizeString = 0, nestedClassRecursion = 0) + * - run 1 + * - JSON Serializer (Map only) throughput = 350.7 MB/s for 34.3 kB per test run (took 9793.0 ms) + * - CMW Serializer (Map only) throughput = 1.7 GB/s for 29.2 kB per test run (took 1755.0 ms) + * - CmwLight Serializer (Map only) throughput = 6.7 GB/s for 29.2 kB per test run (took 437.0 ms) + * - IO Serializer (Map only) throughput = 6.1 GB/s for 29.7 kB per test run (took 485.0 ms) + * + * - FlatBuffers (custom FlexBuffers) throughput = 123.1 MB/s for 30.1 kB per test run (took 24467.0 ms) + * - CmwLight Serializer (custom) throughput = 3.9 GB/s for 29.2 kB per test run (took 751.0 ms) + * - IO Serializer (custom) throughput = 3.8 GB/s for 29.7 kB per test run (took 782.0 ms) + * + * - JSON Serializer (POJO) throughput = 31.7 MB/s for 34.3 kB per test run (took 108415.0 ms) + * - CMW Serializer (POJO) throughput = 1.5 GB/s for 29.2 kB per test run (took 1924.0 ms) + * - CmwLight Serializer (POJO) throughput = 3.5 GB/s for 29.1 kB per test run (took 824.0 ms) + * - IO Serializer (POJO) throughput = 3.4 GB/s for 29.7 kB per test run (took 870.0 ms) + * + * @author rstein + */ +public class SerialiserQuickBenchmark { // NOPMD - nomen est omen + private static final Logger LOGGER = LoggerFactory.getLogger(SerialiserQuickBenchmark.class); + private static final TestDataClass inputObject = new TestDataClass(10, 100, 1); + private static final TestDataClass outputObject = new TestDataClass(-1, -1, 0); + + public static String humanReadableByteCount(final long bytes, final boolean si) { + final int unit = si ? 1000 : 1024; + if (bytes < unit) { + return bytes + " B"; + } + + final int exp = (int) (Math.log(bytes) / Math.log(unit)); + final String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp - 1) + (si ? "" : "i"); + return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); + } + + public static void main(final String... argv) { + // CmwHelper.checkSerialiserIdentity(inputObject, outputObject); + LOGGER.atInfo().addArgument(CmwLightHelper.checkSerialiserIdentity(inputObject, outputObject)).log("CmwLight serialiser nBytes = {}"); + LOGGER.atInfo().addArgument(CmwLightHelper.checkCustomSerialiserIdentity(inputObject, outputObject)).log("CmwLight (custom) serialiser nBytes = {}"); + + LOGGER.atInfo().addArgument(JsonHelper.checkSerialiserIdentity(inputObject, outputObject)).log("JSON Serialiser serialiser nBytes = {}"); + LOGGER.atInfo().addArgument(JsonHelper.checkCustomSerialiserIdentity(inputObject, outputObject)).log("JSON Serialiser (custom) serialiser nBytes = {}"); + LOGGER.atInfo().addArgument(SerialiserHelper.checkSerialiserIdentity(inputObject, outputObject)).log("generic serialiser nBytes = {}"); + LOGGER.atInfo().addArgument(SerialiserHelper.checkCustomSerialiserIdentity(inputObject, outputObject)).log("generic serialiser (custom) nBytes = {}"); + LOGGER.atInfo().addArgument(FlatBuffersHelper.checkCustomSerialiserIdentity(inputObject, outputObject)).log("flatBuffers serialiser nBytes = {}"); + + // Cmw vs. CmwLight compatibility - requires CMW binary libs + // CmwLightHelper.checkCmwLightVsCmwIdentityForward(inputObject, outputObject); + // CmwLightHelper.checkCmwLightVsCmwIdentityBackward(inputObject, outputObject); + + // optimisation to be enabled if e.g. to protocols that do not support UTF-8 string encoding + // CmwLightHelper.getCmwLightSerialiser().setEnforceSimpleStringEncoding(true); + // SerialiserHelper.getBinarySerialiser().setEnforceSimpleStringEncoding(true); + // SerialiserHelper.getBinarySerialiser().setPutFieldMetaData(false); + + final int nIterations = 100000; + for (int i = 0; i < 10; i++) { + LOGGER.atInfo().addArgument(i).log("run {}"); + // map-only performance + JsonHelper.testSerialiserPerformanceMap(nIterations, inputObject); + // CmwHelper.testSerialiserPerformanceMap(nIterations, inputObject, outputObject); + CmwLightHelper.testSerialiserPerformanceMap(nIterations, inputObject); + SerialiserHelper.testSerialiserPerformanceMap(nIterations, inputObject); + + // custom serialiser performance + JsonHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject); + FlatBuffersHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject); + CmwLightHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject); + SerialiserHelper.testCustomSerialiserPerformance(nIterations, inputObject, outputObject); + + // POJO performance + JsonHelper.testPerformancePojo(nIterations, inputObject, outputObject); + JsonHelper.testPerformancePojoCodeGen(nIterations, inputObject, outputObject); + // CmwHelper.testPerformancePojo(nIterations, inputObject, outputObject); + CmwLightHelper.testPerformancePojo(nIterations, inputObject, outputObject); + SerialiserHelper.testPerformancePojo(nIterations, inputObject, outputObject); + } + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/spi/BinarySerialiserTests.java b/serialiser/src/test/java/io/opencmw/serialiser/spi/BinarySerialiserTests.java new file mode 100644 index 00000000..ebc9c750 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/spi/BinarySerialiserTests.java @@ -0,0 +1,714 @@ +package io.opencmw.serialiser.spi; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.spi.helper.MyGenericClass; +import io.opencmw.serialiser.utils.AssertUtils; + +/** + * + * @author rstein + */ +@SuppressWarnings("PMD.ExcessiveMethodLength") +class BinarySerialiserTests { + private static final int BUFFER_SIZE = 2000; + private static final int ARRAY_DIM_1D = 1; // array dimension + + private void putGenericTestArrays(final BinarySerialiser ioSerialiser) { + ioSerialiser.putHeaderInfo(); + ioSerialiser.putGenericArrayAsPrimitive(DataType.BOOL, new Boolean[] { true, false, true }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.BYTE, new Byte[] { (byte) 1, (byte) 0, (byte) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.CHAR, new Character[] { (char) 1, (char) 0, (char) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.SHORT, new Short[] { (short) 1, (short) 0, (short) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.INT, new Integer[] { 1, 0, 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.LONG, new Long[] { 1L, 0L, 2L }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.FLOAT, new Float[] { (float) 1, (float) 0, (float) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.DOUBLE, new Double[] { (double) 1, (double) 0, (double) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.STRING, new String[] { "1.0", "0.0", "2.0" }, 3); + } + + @DisplayName("basic primitive array writer tests") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testBasicInterfacePrimitiveArrays(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); // TODO: generalise to IoBuffer + + Deque positionBefore = new LinkedList<>(); + Deque positionAfter = new LinkedList<>(); + + // add primitive array types + positionBefore.add(buffer.position()); + ioSerialiser.put("boolean", new boolean[] { true, false }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("byte", new byte[] { (byte) 42, (byte) 42 }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("short", new short[] { (short) 43, (short) 43 }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("int", new int[] { 44, 44 }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("long", new long[] { (long) 45, (long) 45 }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("float", new float[] { 1.0f, 1.0f }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("double", new double[] { 3.0, 3.0 }); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("string", new String[] { "test", "test" }); + positionAfter.add(buffer.position()); + + WireDataFieldDescription header; + int positionAfterFieldHeader; + long skipNBytes; + int[] dims; + // check primitive types + buffer.flip(); + assertEquals(0, buffer.position(), "initial buffer position"); + + // boolean + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("boolean", header.getFieldName(), "boolean field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new boolean[] { true, false }, ioSerialiser.getBuffer().getBooleanArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new boolean[] { true, false }, ioSerialiser.getBooleanArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // byte + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("byte", header.getFieldName(), "byte field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new byte[] { (byte) 42, (byte) 42 }, ioSerialiser.getBuffer().getByteArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new byte[] { (byte) 42, (byte) 42 }, ioSerialiser.getByteArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // short + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("short", header.getFieldName(), "short field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new short[] { (short) 43, (short) 43 }, ioSerialiser.getBuffer().getShortArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new short[] { (short) 43, (short) 43 }, ioSerialiser.getShortArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // int + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("int", header.getFieldName(), "int field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new int[] { 44, 44 }, ioSerialiser.getBuffer().getIntArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 44, 44 }, ioSerialiser.getIntArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // long + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("long", header.getFieldName(), "long field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new long[] { 45, 45 }, ioSerialiser.getBuffer().getLongArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new long[] { 45, 45 }, ioSerialiser.getLongArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // float + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("float", header.getFieldName(), "float field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new float[] { 1.0f, 1.0f }, ioSerialiser.getBuffer().getFloatArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new float[] { 1.0f, 1.0f }, ioSerialiser.getFloatArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // double + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("double", header.getFieldName(), "double field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new double[] { 3.0, 3.0 }, ioSerialiser.getBuffer().getDoubleArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new double[] { 3.0, 3.0 }, ioSerialiser.getDoubleArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // string + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("string", header.getFieldName(), "string field name retrieval"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + assertEquals(positionAfterFieldHeader, header.getDataStartPosition(), "data start position"); + assertEquals(positionAfter.peekFirst(), header.getDataStartPosition() + header.getDataSize(), "data end skip address"); + dims = ioSerialiser.getArraySizeDescriptor(); + assertEquals(ARRAY_DIM_1D, dims.length, "dimension"); + assertEquals(2, dims[0], "array size"); + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new int[] { 2 }, ioSerialiser.getArraySizeDescriptor()); // manual dimension check + assertArrayEquals(new String[] { "test", "test" }, ioSerialiser.getBuffer().getStringArray()); // get data from IoBuffer + buffer.position(header.getDataStartPosition()); // return to original data start + assertArrayEquals(new String[] { "test", "test" }, ioSerialiser.getStringArray()); // get data from IoSerialiser + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertEquals(skipNBytes, buffer.position() - positionAfterFieldHeader, "actual number of bytes skipped"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + } + + @DisplayName("basic primitive writer tests") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testBasicInterfacePrimitives(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); // TODO: generalise to IoBuffer + + Deque positionBefore = new LinkedList<>(); + Deque positionAfter = new LinkedList<>(); + + // add primitive types + positionBefore.add(buffer.position()); + ioSerialiser.put("boolean", true); + positionAfter.add(buffer.position()); + positionBefore.add(buffer.position()); + ioSerialiser.put("boolean", false); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("byte", (byte) 42); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("short", (short) 43); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("int", 44); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("long", 45L); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("float", 1.0f); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("double", 3.0); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("string", "test"); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("string", ""); + positionAfter.add(buffer.position()); + + positionBefore.add(buffer.position()); + ioSerialiser.put("string", (String) null); + positionAfter.add(buffer.position()); + + WireDataFieldDescription header; + // check primitive types + buffer.flip(); + assertEquals(0, buffer.position(), "initial buffer position"); + + // boolean + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("boolean", header.getFieldName(), "boolean field name retrieval"); + assertTrue(ioSerialiser.getBuffer().getBoolean(), "byte retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("boolean", header.getFieldName(), "boolean field name retrieval"); + assertFalse(ioSerialiser.getBuffer().getBoolean(), "byte retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // byte + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("byte", header.getFieldName(), "boolean field name retrieval"); + assertEquals(42, ioSerialiser.getBuffer().getByte(), "byte retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // short + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("short", header.getFieldName(), "short field name retrieval"); + assertEquals(43, ioSerialiser.getBuffer().getShort(), "short retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // int + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("int", header.getFieldName(), "int field name retrieval"); + assertEquals(44, ioSerialiser.getBuffer().getInt(), "int retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // long + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("long", header.getFieldName(), "long field name retrieval"); + assertEquals(45, ioSerialiser.getBuffer().getLong(), "long retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // float + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("float", header.getFieldName(), "float field name retrieval"); + assertEquals(1.0f, ioSerialiser.getBuffer().getFloat(), "float retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // double + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("double", header.getFieldName(), "double field name retrieval"); + assertEquals(3.0, ioSerialiser.getBuffer().getDouble(), "double retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // string + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("string", header.getFieldName(), "string field name retrieval"); + assertEquals("test", ioSerialiser.getBuffer().getString(), "string retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("string", header.getFieldName(), "string field name retrieval"); + assertEquals("", ioSerialiser.getBuffer().getString(), "string retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("string", header.getFieldName(), "string field name retrieval"); + assertEquals("", ioSerialiser.getBuffer().getString(), "string retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + } + + @DisplayName("basic tests") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testHeaderAndSpecialItems(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); + + Deque positionBefore = new LinkedList<>(); + Deque positionAfter = new LinkedList<>(); + + // add header info + positionBefore.add(buffer.position()); + ioSerialiser.putHeaderInfo(); + positionAfter.add(buffer.position()); + + // add start marker + positionBefore.add(buffer.position()); + final String dataStartMarkerName = "StartMarker"; + final WireDataFieldDescription dataStartMarker = new WireDataFieldDescription(ioSerialiser, null, dataStartMarkerName.hashCode(), dataStartMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(dataStartMarker); + positionAfter.add(buffer.position()); + + // add Collection - List + final List list = Arrays.asList(1, 2, 3); + positionBefore.add(buffer.position()); + ioSerialiser.put("collection", list, Integer.class); + positionAfter.add(buffer.position()); + + // add Collection - Set + final Set set = Set.of(1, 2, 3); + positionBefore.add(buffer.position()); + ioSerialiser.put("set", set, Integer.class); + positionAfter.add(buffer.position()); + + // add Collection - Queue + final Queue queue = new LinkedList<>(Arrays.asList(1, 2, 3)); + positionBefore.add(buffer.position()); + ioSerialiser.put("queue", queue, Integer.class); + positionAfter.add(buffer.position()); + + // add Map + final Map map = new HashMap<>(); + list.forEach(item -> map.put(item, "Item#" + item.toString())); + positionBefore.add(buffer.position()); + ioSerialiser.put("map", map, Integer.class, String.class); + positionAfter.add(buffer.position()); + + // add Enum + positionBefore.add(buffer.position()); + ioSerialiser.put("enum", DataType.ENUM); + positionAfter.add(buffer.position()); + + // add end marker + positionBefore.add(buffer.position()); + final String dataEndMarkerName = "EndMarker"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); + positionAfter.add(buffer.position()); + + buffer.flip(); + + WireDataFieldDescription header; + int positionAfterFieldHeader; + long skipNBytes; + // check types + assertEquals(0, buffer.position(), "initial buffer position"); + + // header info + assertEquals(positionBefore.removeFirst(), buffer.position()); + ProtocolInfo headerInfo = ioSerialiser.checkHeaderInfo(); + assertNotEquals(headerInfo, new Object()); // silly comparison for coverage reasons + assertNotNull(headerInfo); + assertEquals(BinarySerialiser.PROTOCOL_NAME, headerInfo.getProducerName()); + assertEquals(BinarySerialiser.VERSION_MAJOR, headerInfo.getVersionMajor()); + assertEquals(BinarySerialiser.VERSION_MINOR, headerInfo.getVersionMinor()); + assertEquals(BinarySerialiser.VERSION_MICRO, headerInfo.getVersionMicro()); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // start marker + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("StartMarker", header.getFieldName(), "StartMarker type retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // Collections - List + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("collection", header.getFieldName(), "List field name"); + assertEquals(DataType.LIST, header.getDataType(), "List - type ID"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertNotNull(positionAfter.peekFirst()); + assertEquals(positionAfter.peekFirst() - positionAfterFieldHeader, skipNBytes, "buffer skip address"); + assertFalse(header.getDataType().isScalar()); + assertEquals(ARRAY_DIM_1D, ioSerialiser.getBuffer().getInt(), "dimension"); + assertEquals(3, ioSerialiser.getBuffer().getInt(), "array size"); + buffer.position(header.getDataStartPosition()); + final int readPosition = buffer.position(); + Collection retrievedCollection = ioSerialiser.getCollection(null); + assertNotNull(retrievedCollection, "retrieved collection not null"); + assertEquals(list, retrievedCollection); + assertEquals(buffer.position(), header.getDataStartPosition() + header.getDataSize(), "buffer position data end"); + // check for specific List interface + buffer.position(readPosition); + List retrievedList = ioSerialiser.getList(null); + assertNotNull(retrievedList, "retrieved collection List not null"); + assertEquals(list, retrievedList); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // Collections - Set + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("set", header.getFieldName(), "Set field name"); + assertEquals(DataType.SET, header.getDataType(), "Set - type ID"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertNotNull(positionAfter.peekFirst()); + assertEquals(positionAfter.peekFirst() - positionAfterFieldHeader, skipNBytes, "buffer skip address"); + assertFalse(header.getDataType().isScalar()); + assertEquals(ARRAY_DIM_1D, ioSerialiser.getBuffer().getInt(), "dimension"); + assertEquals(3, ioSerialiser.getBuffer().getInt(), "array size"); + buffer.position(header.getDataStartPosition()); + Collection retrievedSet = ioSerialiser.getSet(null); + assertNotNull(retrievedSet, "retrieved set not null"); + assertEquals(set, retrievedSet); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // Collections - Queue + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("queue", header.getFieldName(), "Queue field name"); + assertEquals(DataType.QUEUE, header.getDataType(), "Queue - type ID"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertNotNull(positionAfter.peekFirst()); + assertEquals(positionAfter.peekFirst() - positionAfterFieldHeader, skipNBytes, "buffer skip address"); + assertFalse(header.getDataType().isScalar()); + assertEquals(ARRAY_DIM_1D, ioSerialiser.getBuffer().getInt(), "dimension"); + assertEquals(3, ioSerialiser.getBuffer().getInt(), "array size"); + buffer.position(header.getDataStartPosition()); + Queue retrievedQueue = ioSerialiser.getQueue(null); + assertNotNull(retrievedQueue, "retrieved set not null"); + // assertEquals(queue, retrievedQueue); // N.B. no direct comparison possible -> only partial Queue interface overlapp + while (!queue.isEmpty() && !retrievedQueue.isEmpty()) { + assertEquals(queue.poll(), retrievedQueue.poll()); + } + assertEquals(0, queue.size(), "reference queue empty"); + assertEquals(0, retrievedQueue.size(), "retrieved queue empty"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // retrieve Map + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("map", header.getFieldName(), "Map field name"); + assertEquals(DataType.MAP, header.getDataType(), "Map - type ID"); + positionAfterFieldHeader = buffer.position(); // actual buffer position after having read the field header + skipNBytes = header.getDataSize(); // number of bytes to be skipped till end of this data chunk + assertNotNull(positionAfter.peekFirst()); + assertEquals(positionAfter.peekFirst() - positionAfterFieldHeader, skipNBytes, "buffer skip address"); + assertFalse(header.getDataType().isScalar()); + assertEquals(ARRAY_DIM_1D, ioSerialiser.getBuffer().getInt(), "dimension"); + assertEquals(3, ioSerialiser.getBuffer().getInt(), "array size"); + buffer.position(header.getDataStartPosition()); + Map retrievedMap = ioSerialiser.getMap(null); + assertNotNull(retrievedMap, "retrieved set not null"); + assertEquals(map, retrievedMap); // N.B. no direct comparison possible -> only partial Queue interface overlapp + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // enum + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("enum", header.getFieldName(), "enum type retrieval"); + buffer.position(header.getDataStartPosition()); + assertDoesNotThrow(ioSerialiser::getEnumTypeList); //skips enum info + buffer.position(header.getDataStartPosition()); + assertEquals(DataType.ENUM, ioSerialiser.getEnum(DataType.OTHER), "enum retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + + // end marker + assertEquals(positionBefore.removeFirst(), buffer.position()); + header = ioSerialiser.getFieldHeader(); + assertEquals("EndMarker", header.getFieldName(), "EndMarker type retrieval"); + assertEquals(positionAfter.removeFirst(), buffer.position()); + } + + @Test + void testIdentityGenericObject() { + // Simple tests to verify that the equals and hashCode functions of 'MyGenericClass' work as expected + final MyGenericClass rootObject1 = new MyGenericClass(); + final MyGenericClass rootObject2 = new MyGenericClass(); + MyGenericClass.setVerboseChecks(false); + + assertNotNull(rootObject1.toString()); + assertNotNull(rootObject1.boxedPrimitives.toString()); + + assertEquals(rootObject1, rootObject2); + + rootObject1.modifyValues(); + assertNotEquals(rootObject1, rootObject2); + rootObject2.modifyValues(); + assertEquals(rootObject1, rootObject2); + + rootObject1.boxedPrimitives.modifyValues(); + assertNotEquals(rootObject1, rootObject2); + rootObject2.boxedPrimitives.modifyValues(); + assertEquals(rootObject1, rootObject2); + + rootObject1.arrays.modifyValues(); + assertNotEquals(rootObject1, rootObject2); + rootObject2.arrays.modifyValues(); + assertEquals(rootObject1, rootObject2); + + rootObject1.objArrays.modifyValues(); + assertNotEquals(rootObject1, rootObject2); + rootObject2.objArrays.modifyValues(); + assertEquals(rootObject1, rootObject2); + } + + @DisplayName("basic primitive array writer tests") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testParseIoStream(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); // a bit larger buffer since we test more cases at once + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); + + ioSerialiser.putHeaderInfo(); // add header info + + // add some primitives + ioSerialiser.put("boolean", true); + ioSerialiser.put("byte", (byte) 42); + ioSerialiser.put("short", (short) 42); + ioSerialiser.put("int", 42); + ioSerialiser.put("long", 42L); + ioSerialiser.put("float", 42f); + ioSerialiser.put("double", 42); + ioSerialiser.put("string", "string"); + + ioSerialiser.put("boolean[]", new boolean[] { true }, 1); + ioSerialiser.put("byte[]", new byte[] { (byte) 42 }, 1); + ioSerialiser.put("short[]", new short[] { (short) 42 }, 1); + ioSerialiser.put("int[]", new int[] { 42 }, 1); + ioSerialiser.put("long[]", new long[] { 42L }, 1); + ioSerialiser.put("float[]", new float[] { (float) 42 }, 1); + ioSerialiser.put("double[]", new double[] { (double) 42 }, 1); + ioSerialiser.put("string[]", new String[] { "string" }, 1); + + final Collection collection = Arrays.asList(1, 2, 3); + ioSerialiser.put("collection", collection, Integer.class); // add Collection - List + + final List list = Arrays.asList(1, 2, 3); + ioSerialiser.put("list", list, Integer.class); // add Collection - List + + final Set set = Set.of(1, 2, 3); + ioSerialiser.put("set", set, Integer.class); // add Collection - Set + + final Queue queue = new LinkedList<>(Arrays.asList(1, 2, 3)); + ioSerialiser.put("queue", queue, Integer.class); // add Collection - Queue + + final Map map = new HashMap<>(); + list.forEach(item -> map.put(item, "Item#" + item.toString())); + ioSerialiser.put("map", map, Integer.class, String.class); // add Map + + ioSerialiser.put("enum", DataType.ENUM); // add Enum + + // start nested data + final String nestedContextName = "nested context"; + final WireDataFieldDescription nestedContextMarker = new WireDataFieldDescription(ioSerialiser, null, nestedContextName.hashCode(), nestedContextName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(nestedContextMarker); // add start marker + ioSerialiser.put("booleanArray", new boolean[] { true }, 1); + ioSerialiser.put("byteArray", new byte[] { (byte) 0x42 }, 1); + + ioSerialiser.putEndMarker(nestedContextMarker); // add end marker + // end nested data + + final String dataEndMarkerName = "Life is good!"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); // add end marker + + buffer.flip(); + + // and read back streamed items + final WireDataFieldDescription objectRoot = ioSerialiser.parseIoStream(true); + assertNotNull(objectRoot); + // objectRoot.printFieldStructure(); + } + + @DisplayName("test getGenericArrayAsBoxedPrimitive(...) helper method") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testGetGenericArrayAsBoxedPrimitiveHelper(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); // a bit larger buffer since we test more cases at once + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); + + putGenericTestArrays(ioSerialiser); + + buffer.flip(); + + // test conversion to double array + ioSerialiser.checkHeaderInfo(); + assertArrayEquals(new Boolean[] { true, false, true }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.BOOL)); + assertArrayEquals(new Byte[] { (byte) 1.0, (byte) 0.0, (byte) 2.0 }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.BYTE)); + assertArrayEquals(new Character[] { (char) 1.0, (char) 0.0, (char) 2.0 }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.CHAR)); + assertArrayEquals(new Short[] { (short) 1.0, (short) 0.0, (short) 2.0 }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.SHORT)); + assertArrayEquals(new Integer[] { 1, 0, 2 }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.INT)); + assertArrayEquals(new Long[] { (long) 1.0, (long) 0.0, (long) 2.0 }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.LONG)); + assertArrayEquals(new Float[] { 1.0f, 0.0f, 2.0f }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.FLOAT)); + assertArrayEquals(new Double[] { 1.0, 0.0, 2.0 }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.DOUBLE)); + assertArrayEquals(new String[] { "1.0", "0.0", "2.0" }, ioSerialiser.getGenericArrayAsBoxedPrimitive(DataType.STRING)); + } + + @Test + void testMisc() { + final BinarySerialiser ioSerialiser = new BinarySerialiser(new FastByteBuffer(1000)); // TODO: generalise to IoBuffer + final int bufferIncrements = ioSerialiser.getBufferIncrements(); + AssertUtils.gtEqThanZero("bufferIncrements", bufferIncrements); + ioSerialiser.setBufferIncrements(bufferIncrements + 1); + assertEquals(bufferIncrements + 1, ioSerialiser.getBufferIncrements()); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/spi/CmwLightSerialiserTests.java b/serialiser/src/test/java/io/opencmw/serialiser/spi/CmwLightSerialiserTests.java new file mode 100644 index 00000000..b5d0af59 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/spi/CmwLightSerialiserTests.java @@ -0,0 +1,26 @@ +package io.opencmw.serialiser.spi; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import io.opencmw.serialiser.FieldDescription; + +public class CmwLightSerialiserTests { + @Test + public void testCmwData() { + final CmwLightSerialiser serialiser = new CmwLightSerialiser(FastByteBuffer.wrap(new byte[] { + 7, 0, 0, 0, 2, 0, 0, 0, 48, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 49, 0, 7, 1, 0, 0, 0, 0, 2, 0, 0, + 0, 50, 0, 1, 5, 2, 0, 0, 0, 51, 0, 8, 1, 0, 0, 0, 2, 0, 0, 0, 98, 0, 4, 114, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 55, 0, 1, 0, 2, 0, 0, 0, 100, 0, 7, 1, 0, 0, 0, 0, 2, 0, 0, 0, 102, 0, 7, 1, 0, 0, 0, 0 })); + final FieldDescription fieldDescription = serialiser.parseIoStream(true).getChildren().get(0); + // fieldDescription.printFieldStructure(); + assertEquals(1L, ((WireDataFieldDescription) fieldDescription.findChildField("0")).data()); + assertEquals("", ((WireDataFieldDescription) fieldDescription.findChildField("1")).data()); + assertEquals((byte) 5, ((WireDataFieldDescription) fieldDescription.findChildField("2")).data()); + assertEquals(114L, ((WireDataFieldDescription) fieldDescription.findChildField("3").findChildField("b")).data()); + assertEquals((byte) 0, ((WireDataFieldDescription) fieldDescription.findChildField("7")).data()); + assertEquals("", ((WireDataFieldDescription) fieldDescription.findChildField("d")).data()); + assertEquals("", ((WireDataFieldDescription) fieldDescription.findChildField("f")).data()); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/spi/IoBufferTests.java b/serialiser/src/test/java/io/opencmw/serialiser/spi/IoBufferTests.java new file mode 100644 index 00000000..978d382c --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/spi/IoBufferTests.java @@ -0,0 +1,561 @@ +package io.opencmw.serialiser.spi; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.utils.ByteArrayCache; + +/** + * @author rstein + */ +class IoBufferTests { + protected static final boolean[] booleanTestArray = { true, false, true, false }; + protected static final byte[] byteTestArray = { 100, 101, 102, 103, -100, -101, -102, -103 }; + protected static final short[] shortTestArray = { -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5 }; // NOPMD by rstein + protected static final int[] intTestArray = { 5, 4, 3, 2, 1, 0, -1, -2, -3, -4, -5 }; + protected static final long[] longTestArray = { Integer.MAX_VALUE, Integer.MAX_VALUE - 1, -Integer.MAX_VALUE + 2 }; + protected static final float[] floatTestArray = { 1.1e9f, 1.2e9f, 1.3e9f, -1.1e9f, -1.2e9f, -1.3e9f }; + protected static final double[] doubleTestArray = { Float.MAX_VALUE + 1.1e9, Float.MAX_VALUE + 1.2e9, Float.MAX_VALUE + 1.3e9f, -Float.MAX_VALUE - 1.1e9f, -Float.MAX_VALUE - 1.2e9f, Float.MAX_VALUE - 1.3e9f }; + protected static final char[] charTestArray = { 'a', 'b', 'c', 'd' }; + protected static final String[] stringTestArray = { "Is", "this", "the", "real", "life?", "Is", "this", "just", "fantasy?", "", null }; + protected static final String[] stringTestArrayNullAsEmpty = Arrays.stream(stringTestArray).map(s -> s == null ? "" : s).toArray(String[]::new); + private static final int BUFFER_SIZE = 1000; + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { FastByteBuffer.class }) // trim is not implemented for ByteBuffer + @SuppressWarnings("PMD.ExcessiveMethodLength") + void trimTest(final Class bufferClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(10); + + buffer.position(5); + buffer.trim(12); + assertEquals(10, buffer.capacity()); + buffer.trim(7); + assertEquals(7, buffer.capacity()); + buffer.trim(); + assertEquals(5, buffer.capacity()); + buffer.trim(3); + assertEquals(5, buffer.capacity()); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + @SuppressWarnings("PMD.ExcessiveMethodLength") + void primitivesArrays(final Class bufferClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + + assertNotNull(buffer.toString()); + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putBooleanArray(booleanTestArray, booleanTestArray.length)); + assertDoesNotThrow(() -> buffer.putBooleanArray(booleanTestArray, -1)); + assertDoesNotThrow(() -> buffer.putBooleanArray(null, 5)); + buffer.flip(); + assertArrayEquals(booleanTestArray, buffer.getBooleanArray()); + assertArrayEquals(booleanTestArray, buffer.getBooleanArray()); + assertEquals(0, buffer.getBooleanArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putByteArray(byteTestArray, byteTestArray.length)); + assertDoesNotThrow(() -> buffer.putByteArray(byteTestArray, -1)); + assertDoesNotThrow(() -> buffer.putByteArray(null, 5)); + buffer.flip(); + assertArrayEquals(byteTestArray, buffer.getByteArray()); + assertArrayEquals(byteTestArray, buffer.getByteArray()); + assertEquals(0, buffer.getByteArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putShortArray(shortTestArray, shortTestArray.length)); + assertDoesNotThrow(() -> buffer.putShortArray(shortTestArray, -1)); + assertDoesNotThrow(() -> buffer.putShortArray(null, 5)); + buffer.flip(); + assertArrayEquals(shortTestArray, buffer.getShortArray()); + assertArrayEquals(shortTestArray, buffer.getShortArray()); + assertEquals(0, buffer.getShortArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putIntArray(intTestArray, intTestArray.length)); + assertDoesNotThrow(() -> buffer.putIntArray(intTestArray, -1)); + assertDoesNotThrow(() -> buffer.putIntArray(null, 5)); + buffer.flip(); + assertArrayEquals(intTestArray, buffer.getIntArray()); + assertArrayEquals(intTestArray, buffer.getIntArray()); + assertEquals(0, buffer.getIntArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putLongArray(longTestArray, longTestArray.length)); + assertDoesNotThrow(() -> buffer.putLongArray(longTestArray, -1)); + assertDoesNotThrow(() -> buffer.putLongArray(null, 5)); + buffer.flip(); + assertArrayEquals(longTestArray, buffer.getLongArray()); + assertArrayEquals(longTestArray, buffer.getLongArray()); + assertEquals(0, buffer.getLongArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putFloatArray(floatTestArray, floatTestArray.length)); + assertDoesNotThrow(() -> buffer.putFloatArray(floatTestArray, -1)); + assertDoesNotThrow(() -> buffer.putFloatArray(null, 5)); + buffer.flip(); + assertArrayEquals(floatTestArray, buffer.getFloatArray()); + assertArrayEquals(floatTestArray, buffer.getFloatArray()); + assertEquals(0, buffer.getFloatArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putDoubleArray(doubleTestArray, doubleTestArray.length)); + assertDoesNotThrow(() -> buffer.putDoubleArray(doubleTestArray, -1)); + assertDoesNotThrow(() -> buffer.putDoubleArray(null, 5)); + buffer.flip(); + assertArrayEquals(doubleTestArray, buffer.getDoubleArray()); + assertArrayEquals(doubleTestArray, buffer.getDoubleArray()); + assertEquals(0, buffer.getDoubleArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putCharArray(charTestArray, charTestArray.length)); + assertDoesNotThrow(() -> buffer.putCharArray(charTestArray, -1)); + assertDoesNotThrow(() -> buffer.putCharArray(null, 5)); + buffer.flip(); + assertArrayEquals(charTestArray, buffer.getCharArray()); + assertArrayEquals(charTestArray, buffer.getCharArray()); + assertEquals(0, buffer.getCharArray().length); + } + + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putStringArray(stringTestArray, stringTestArray.length)); + assertDoesNotThrow(() -> buffer.putStringArray(stringTestArray, -1)); + assertDoesNotThrow(() -> buffer.putStringArray(null, 5)); + buffer.flip(); + assertArrayEquals(stringTestArrayNullAsEmpty, buffer.getStringArray()); + assertArrayEquals(stringTestArrayNullAsEmpty, buffer.getStringArray()); + assertEquals(0, buffer.getStringArray().length); + } + } + + @Test + void primitivesArraysASCII() { + FastByteBuffer buffer = new FastByteBuffer(BUFFER_SIZE); + + { + final char[] chars = Character.toChars(0x1F701); + final String fourByteCharacter = new String(chars); + String utf8TestString = "Γειά σου Κόσμε! - " + fourByteCharacter + " 語 \u00ea \u00f1 \u00fc + some normal ASCII character"; + buffer.reset(); + assertDoesNotThrow(() -> buffer.putStringArray(stringTestArray, stringTestArray.length)); + assertDoesNotThrow(() -> buffer.putStringArray(stringTestArray, -1)); + assertDoesNotThrow(() -> buffer.putStringArray(null, 5)); + buffer.putString(utf8TestString); + buffer.flip(); + assertArrayEquals(stringTestArrayNullAsEmpty, buffer.getStringArray()); + assertArrayEquals(stringTestArrayNullAsEmpty, buffer.getStringArray()); + assertEquals(0, buffer.getStringArray().length); + assertEquals(utf8TestString, buffer.getString()); + } + + buffer.setEnforceSimpleStringEncoding(true); + { + buffer.reset(); + assertDoesNotThrow(() -> buffer.putStringArray(stringTestArray, stringTestArray.length)); + assertDoesNotThrow(() -> buffer.putStringArray(stringTestArray, -1)); + assertDoesNotThrow(() -> buffer.putStringArray(null, 5)); + buffer.putString("Hello World!"); + buffer.flip(); + assertArrayEquals(stringTestArrayNullAsEmpty, buffer.getStringArray()); + assertArrayEquals(stringTestArrayNullAsEmpty, buffer.getStringArray()); + assertEquals(0, buffer.getStringArray().length); + assertEquals("Hello World!", buffer.getString()); + } + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void primitivesMixed(final Class bufferClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final long largeLong = (long) Integer.MAX_VALUE + (long) 10; + + buffer.reset(); + buffer.putBoolean(true); + buffer.putBoolean(false); + buffer.putByte((byte) 0xFE); + buffer.putShort((short) 43); + buffer.putInt(1025); + buffer.putLong(largeLong); + buffer.putFloat(1.3e10f); + buffer.putDouble(1.3e10f); + buffer.putChar('@'); + buffer.putChar((char) 513); + buffer.putStringISO8859("Hello World!"); + buffer.putString("Γειά σου Κόσμε!"); + final long position = buffer.position(); + + // return to start position + buffer.flip(); + assertTrue(buffer.getBoolean()); + assertFalse(buffer.getBoolean()); + assertEquals(buffer.getByte(), (byte) 0xFE); + assertEquals(buffer.getShort(), (short) 43); + assertEquals(1025, buffer.getInt()); + assertEquals(buffer.getLong(), largeLong); + assertEquals(1.3e10f, buffer.getFloat()); + assertEquals(1.3e10f, buffer.getDouble()); + assertEquals('@', buffer.getChar()); + assertEquals((char) 513, buffer.getChar()); + assertEquals("Hello World!", buffer.getStringISO8859()); + assertEquals("Γειά σου Κόσμε!", buffer.getString()); + assertEquals(buffer.position(), position); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void primitivesSimple(final Class bufferClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + buffer.reset(); + buffer.putBoolean(true); + buffer.flip(); + assertTrue(buffer.getBoolean()); + + buffer.reset(); + buffer.putBoolean(false); + buffer.flip(); + assertFalse(buffer.getBoolean()); + + buffer.reset(); + buffer.putByte((byte) 0xFE); + buffer.flip(); + assertEquals(buffer.getByte(), (byte) 0xFE); + + buffer.reset(); + buffer.putShort((short) 43); + buffer.flip(); + assertEquals(buffer.getShort(), (short) 43); + + buffer.reset(); + buffer.putInt(1025); + buffer.flip(); + assertEquals(1025, buffer.getInt()); + + buffer.reset(); + final long largeLong = (long) Integer.MAX_VALUE + (long) 10; + buffer.putLong(largeLong); + buffer.flip(); + assertEquals(buffer.getLong(), largeLong); + + buffer.reset(); + buffer.putFloat(1.3e10f); + buffer.flip(); + assertEquals(1.3e10f, buffer.getFloat()); + + buffer.reset(); + buffer.putDouble(1.3e10f); + buffer.flip(); + assertEquals(1.3e10f, buffer.getDouble()); + + buffer.reset(); + buffer.putChar('@'); + buffer.flip(); + assertEquals('@', buffer.getChar()); + + buffer.reset(); + buffer.putChar((char) 513); + buffer.flip(); + assertEquals((char) 513, buffer.getChar()); + + buffer.reset(); + buffer.putString("Hello World!"); + buffer.flip(); + assertEquals("Hello World!", buffer.getString()); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void primitivesSimpleInPlace(final Class bufferClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + buffer.reset(); + + buffer.putBoolean(0, true); + assertTrue(buffer.getBoolean(0)); + + buffer.reset(); + buffer.putBoolean(0, false); + assertFalse(buffer.getBoolean(0)); + + buffer.putByte(1, (byte) 0xFE); + assertEquals(buffer.getByte(1), (byte) 0xFE); + + buffer.putShort(2, (short) 43); + assertEquals(buffer.getShort(2), (short) 43); + + buffer.putInt(3, 1025); + assertEquals(1025, buffer.getInt(3)); + + final long largeLong = (long) Integer.MAX_VALUE + (long) 10; + buffer.putLong(4, largeLong); + assertEquals(buffer.getLong(4), largeLong); + + buffer.putFloat(5, 1.3e10f); + assertEquals(1.3e10f, buffer.getFloat(5)); + + buffer.putDouble(6, 1.3e10f); + assertEquals(1.3e10f, buffer.getDouble(6)); + + buffer.putChar(7, '@'); + assertEquals('@', buffer.getChar(7)); + + buffer.putChar(7, (char) 513); + assertEquals((char) 513, buffer.getChar(7)); + + buffer.putString(8, "Hello World!"); + assertEquals("Hello World!", buffer.getString(8)); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void indexManipulations(final Class bufferClass) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, InstantiationException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(10); + + assertEquals(0, buffer.position()); + assertEquals(10, buffer.limit()); + assertEquals(10, buffer.capacity()); + + assertThrows(IllegalArgumentException.class, () -> buffer.limit(11)); // limit > capacity + buffer.position(5); + buffer.limit(8); + assertEquals(5, buffer.position()); + assertEquals(8, buffer.limit()); + assertEquals(10, buffer.capacity()); + assertTrue(buffer.hasRemaining()); + assertEquals(3, buffer.remaining()); + + buffer.flip(); + assertEquals(0, buffer.position()); + assertEquals(5, buffer.limit()); + assertEquals(10, buffer.capacity()); + + assertThrows(IllegalArgumentException.class, () -> buffer.position(6)); // pos > limit + buffer.position(4); + buffer.limit(3); + assertEquals(3, buffer.position()); + assertEquals(3, buffer.limit()); + assertEquals(10, buffer.capacity()); + assertFalse(buffer.hasRemaining()); + + buffer.reset(); + } + + @Test + void testFastByteBufferAllocators() { + { + FastByteBuffer buffer = new FastByteBuffer(); + assertTrue(buffer.capacity() > 0); + assertEquals(0, buffer.position()); + assertEquals(buffer.limit(), buffer.capacity()); + buffer.limit(buffer.capacity() - 2); + assertEquals(buffer.limit(), (buffer.capacity() - 2)); + assertFalse(buffer.isReadOnly()); + } + + { + FastByteBuffer buffer = new FastByteBuffer(500); + assertEquals(500, buffer.capacity()); + } + + { + FastByteBuffer buffer = new FastByteBuffer(new byte[1000], 500); + assertEquals(1000, buffer.capacity()); + assertEquals(500, buffer.limit()); + assertThrows(IllegalArgumentException.class, () -> new FastByteBuffer(new byte[5], 10)); + } + + { + FastByteBuffer buffer = FastByteBuffer.wrap(byteTestArray); + assertArrayEquals(byteTestArray, buffer.elements()); + } + } + + @Test + void testFastByteBufferResizing() { + FastByteBuffer buffer = new FastByteBuffer(300, true, new ByteArrayCache()); + assertTrue(buffer.isAutoResize()); + assertNotNull(buffer.getByteArrayCache()); + assertEquals(0, buffer.getByteArrayCache().size()); + assertEquals(300, buffer.capacity()); + + buffer.limit(200); // shift limit to index 200 + assertEquals(200, buffer.remaining()); // N.B. == 200 - pos (0); + + buffer.ensureAdditionalCapacity(200); // should be NOP + assertEquals(200, buffer.remaining()); + assertEquals(300, buffer.capacity()); + + buffer.forceCapacity(300, 0); // does no reallocation but moves limit to end + assertEquals(300, buffer.remaining()); // N.B. == 200 - pos (0); + + buffer.ensureCapacity(400); + assertThat(buffer.capacity(), Matchers.greaterThan(400)); + + buffer.putByteArray(new byte[100], 100); + // N.B. int (4 bytes) for array size, n*4 Bytes for actual array + final long sizeArray = (FastByteBuffer.SIZE_OF_INT + 100 * FastByteBuffer.SIZE_OF_BYTE); + assertEquals(104, sizeArray); + assertEquals(sizeArray, buffer.position()); + + assertThat(buffer.capacity(), Matchers.greaterThan(400)); + buffer.trim(); + assertEquals(buffer.capacity(), buffer.position()); + + buffer.ensureCapacity(500); + buffer.trim(333); + assertEquals(333, buffer.capacity()); + + buffer.position(0); + assertEquals(0, buffer.position()); + + buffer.trim(); + assertFalse(buffer.hasRemaining()); + buffer.ensureAdditionalCapacity(100); + assertTrue(buffer.hasRemaining()); + assertThat(buffer.capacity(), Matchers.greaterThan(1124)); + + buffer.limit(50); + buffer.clear(); + assertEquals(0, buffer.position()); + assertEquals(buffer.limit(), buffer.capacity()); + + // test resize related getters and setters + buffer.setAutoResize(false); + assertFalse(buffer.isAutoResize()); + buffer.setByteArrayCache(null); + assertNull(buffer.getByteArrayCache()); + } + + @Test + void testFastByteBufferOutOfBounds() { + final FastByteBuffer buffer = FastByteBuffer.wrap(new byte[50]); + // test single getters + buffer.position(47); + assertThrows(IndexOutOfBoundsException.class, buffer::getInt); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getInt(47)); + assertThrows(IndexOutOfBoundsException.class, buffer::getLong); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getLong(44)); + assertThrows(IndexOutOfBoundsException.class, buffer::getDouble); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getDouble(47)); + assertThrows(IndexOutOfBoundsException.class, buffer::getFloat); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getFloat(47)); + buffer.position(49); + assertThrows(IndexOutOfBoundsException.class, buffer::getShort); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getShort(49)); + buffer.position(50); + assertThrows(IndexOutOfBoundsException.class, buffer::getBoolean); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getBoolean(50)); + assertThrows(IndexOutOfBoundsException.class, buffer::getByte); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getByte(50)); + // test array getters + // INT + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putIntArray(new int[50], 50)); + assertEquals(0, buffer.position()); + buffer.putIntArray(new int[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_INT * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getIntArray(null, 5)); + // LONG + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putLongArray(new long[50], 50)); + assertEquals(0, buffer.position()); + buffer.putLongArray(new long[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_LONG * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getLongArray(null, 5)); + // SHORT + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putShortArray(new short[50], 50)); + assertEquals(0, buffer.position()); + buffer.putShortArray(new short[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_SHORT * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getShortArray(null, 5)); + // CHAR + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putCharArray(new char[50], 50)); + assertEquals(0, buffer.position()); + buffer.putCharArray(new char[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_CHAR * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getCharArray(null, 5)); + // BOOLEAN + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putBooleanArray(new boolean[51], 51)); + assertEquals(0, buffer.position()); + buffer.putBooleanArray(new boolean[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_BOOLEAN * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getBooleanArray(null, 5)); + // BYTE + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putByteArray(new byte[51], 51)); + assertEquals(0, buffer.position()); + buffer.putByteArray(new byte[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_BYTE * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getByteArray(null, 5)); + // DOUBLE + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putDoubleArray(new double[50], 50)); + assertEquals(0, buffer.position()); + buffer.putDoubleArray(new double[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_DOUBLE * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getDoubleArray(null, 5)); + // FLOAT + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putFloatArray(new float[50], 50)); + assertEquals(0, buffer.position()); + buffer.putFloatArray(new float[5], 5); + buffer.flip(); + buffer.limit(FastByteBuffer.SIZE_OF_FLOAT * 4); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getFloatArray(null, 5)); + // STRING + buffer.reset(); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.putStringArray(new String[50], 50)); + assertEquals(0, buffer.position()); + buffer.putStringArray(new String[5], 5); + buffer.flip(); + buffer.limit(10); + assertThrows(IndexOutOfBoundsException.class, () -> buffer.getStringArray(null, 4)); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/spi/JsonSerialiserTests.java b/serialiser/src/test/java/io/opencmw/serialiser/spi/JsonSerialiserTests.java new file mode 100644 index 00000000..443fe9bc --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/spi/JsonSerialiserTests.java @@ -0,0 +1,305 @@ +package io.opencmw.serialiser.spi; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.*; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.IoBuffer; + +/** + * + * @author rstein + */ +@SuppressWarnings("PMD.ExcessiveMethodLength") +class JsonSerialiserTests { + private static final Logger LOGGER = LoggerFactory.getLogger(JsonSerialiser.class); + private static final int BUFFER_SIZE = 2000; + + @DisplayName("basic tests") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testHeaderAndSpecialItems(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + final JsonSerialiser ioSerialiser = new JsonSerialiser(buffer); + + // add header info + ioSerialiser.putHeaderInfo(); + // add start marker + final String dataStartMarkerName = "StartMarker"; + final WireDataFieldDescription dataStartMarker = new WireDataFieldDescription(ioSerialiser, null, dataStartMarkerName.hashCode(), dataStartMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(dataStartMarker); + // add Collection - List + final List list = Arrays.asList(1, 2, 3); + ioSerialiser.put("collection", list, Integer.class); + // add Collection - Set + final Set set = Set.of(1, 2, 3); + ioSerialiser.put("set", set, Integer.class); + // add Collection - Queue + final Queue queue = new LinkedList<>(Arrays.asList(1, 2, 3)); + ioSerialiser.put("queue", queue, Integer.class); + // add Map + final Map map = new HashMap<>(); + list.forEach(item -> map.put(item, "Item#" + item.toString())); + ioSerialiser.put("map", map, Integer.class, String.class); + // add Enum + ioSerialiser.put("enum", DataType.ENUM); + // add end marker + final String dataEndMarkerName = "EndMarker"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); // end start marker + ioSerialiser.putEndMarker(dataEndMarker); // end header info + + buffer.flip(); + + final String result = new String(Arrays.copyOfRange(buffer.elements(), 0, buffer.limit())); + LOGGER.atDebug().addArgument(result).log("serialised:\n{}"); + + final Iterator lines = result.lines().iterator(); + assertEquals("{", lines.next()); + assertEquals(" \"StartMarker\": {", lines.next()); + assertEquals(" \"collection\": [1, 2, 3],", lines.next()); + assertTrue(lines.next().matches(" {4}\"set\": \\[[123], [123], [123]],")); + assertEquals(" \"queue\": [1, 2, 3],", lines.next()); + assertEquals(" \"map\": {\"1\": \"Item#1\", \"2\": \"Item#2\", \"3\": \"Item#3\"},", lines.next()); + assertEquals(" \"enum\": \"ENUM\"", lines.next()); + assertEquals(" }", lines.next()); + assertEquals("", lines.next()); + assertEquals("}", lines.next()); + assertFalse(lines.hasNext()); + // check types + assertEquals(0, buffer.position(), "initial buffer position"); + + // header info + ProtocolInfo headerInfo = ioSerialiser.checkHeaderInfo(); + assertNotEquals(headerInfo, new Object()); // silly comparison for coverage reasons + assertNotNull(headerInfo); + assertEquals(JsonSerialiser.class.getCanonicalName(), headerInfo.getProducerName()); + assertEquals(1, headerInfo.getVersionMajor()); + assertEquals(0, headerInfo.getVersionMinor()); + assertEquals(0, headerInfo.getVersionMicro()); + } + + @DisplayName("basic primitive array writer tests") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testParseIoStream(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { //NOSONAR NOPMD + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); // a bit larger buffer since we test more cases at once + final JsonSerialiser ioSerialiser = new JsonSerialiser(buffer); + + ioSerialiser.putHeaderInfo(); // add header info + + // add some primitives + ioSerialiser.put("boolean", true); + ioSerialiser.put("byte", (byte) 42); + ioSerialiser.put("char", (char) 40d); + ioSerialiser.put("short", (short) 42); + ioSerialiser.put("int", 42); + ioSerialiser.put("long", 42L); + ioSerialiser.put("float", 42f); + ioSerialiser.put("double", 42); + ioSerialiser.put("string", "string"); + + ioSerialiser.put("boolean[]", new boolean[] { true }, 1); + ioSerialiser.put("byte[]", new byte[] { (byte) 42 }, 1); + ioSerialiser.put("char[]", new char[] { (char) 40 }, 1); + ioSerialiser.put("short[]", new short[] { (short) 42 }, 1); + ioSerialiser.put("int[]", new int[] { 42 }, 1); + ioSerialiser.put("long[]", new long[] { 42L }, 1); + ioSerialiser.put("float[]", new float[] { (float) 42 }, 1); + ioSerialiser.put("double[]", new double[] { (double) 42 }, 1); + ioSerialiser.put("string[]", new String[] { "string" }, 1); + + final Collection collection = Arrays.asList(1, 2, 3); + ioSerialiser.put("collection", collection, Integer.class); // add Collection - List + + final List list = Arrays.asList(1, 2, 3); + ioSerialiser.put("list", list, Integer.class); // add Collection - List + + final Set set = Set.of(1, 2, 3); + ioSerialiser.put("set", set, Integer.class); // add Collection - Set + + final Queue queue = new LinkedList<>(Arrays.asList(1, 2, 3)); + ioSerialiser.put("queue", queue, Integer.class); // add Collection - Queue + + final Map map = new HashMap<>(); + list.forEach(item -> map.put(item, "Item#" + item.toString())); + ioSerialiser.put("map", map, Integer.class, String.class); // add Map + + // ioSerialiser.put("enum", DataType.ENUM); // enums cannot be read back in, because there is not type information + + // start nested data + final String nestedContextName = "nested context"; + final WireDataFieldDescription nestedContextMarker = new WireDataFieldDescription(ioSerialiser, null, nestedContextName.hashCode(), nestedContextName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(nestedContextMarker); // add start marker + ioSerialiser.put("booleanArray", new boolean[] { true }, 1); + ioSerialiser.put("byteArray", new byte[] { (byte) 0x42 }, 1); + + ioSerialiser.putEndMarker(nestedContextMarker); // add end marker + // end nested data + + final String dataEndMarkerName = "Life is good!"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); // add end marker + + buffer.flip(); + + final String result = new String(Arrays.copyOfRange(buffer.elements(), 0, buffer.limit())); + LOGGER.atDebug().addArgument(result).log("serialised:\n{}"); + + final Iterator lines = result.lines().iterator(); + assertEquals("{", lines.next()); + assertEquals(" \"boolean\": true,", lines.next()); + assertEquals(" \"byte\": 42,", lines.next()); + assertEquals(" \"char\": 40,", lines.next()); + assertEquals(" \"short\": 42,", lines.next()); + assertEquals(" \"int\": 42,", lines.next()); + assertEquals(" \"long\": 42,", lines.next()); + assertEquals(" \"float\": 42.0,", lines.next()); + assertEquals(" \"double\": 42,", lines.next()); + assertEquals(" \"string\": \"string\",", lines.next()); + assertEquals(" \"boolean[]\": [true],", lines.next()); + assertEquals(" \"byte[]\": [42],", lines.next()); + assertEquals(" \"char[]\": [40],", lines.next()); + assertEquals(" \"short[]\": [42],", lines.next()); + assertEquals(" \"int[]\": [42],", lines.next()); + assertEquals(" \"long[]\": [42],", lines.next()); + assertEquals(" \"float[]\": [42.0],", lines.next()); + assertEquals(" \"double[]\": [42.0],", lines.next()); + assertEquals(" \"string[]\": [\"string\"],", lines.next()); + assertEquals(" \"collection\": [1, 2, 3],", lines.next()); + assertEquals(" \"list\": [1, 2, 3],", lines.next()); + assertTrue(lines.next().matches(" {2}\"set\": \\[[123], [123], [123]],")); + assertEquals(" \"queue\": [1, 2, 3],", lines.next()); + assertEquals(" \"map\": {\"1\": \"Item#1\", \"2\": \"Item#2\", \"3\": \"Item#3\"},", lines.next()); + // assertEquals(" \"enum\": ENUM,", lines.next()); + assertEquals(" \"nested context\": {", lines.next()); + assertEquals(" \"booleanArray\": [true],", lines.next()); + assertEquals(" \"byteArray\": [66]", lines.next()); + assertEquals(" }", lines.next()); + assertEquals("", lines.next()); + assertEquals("}", lines.next()); + assertFalse(lines.hasNext()); + + // and read back streamed items. Note that types get widened, arrays -> list etc due to type info lost + final WireDataFieldDescription objectRoot = ioSerialiser.parseIoStream(true); + assertNotNull(objectRoot); + objectRoot.printFieldStructure(); + } + @Test + @DisplayName("Simple Object in List") + void testObjectAlongPrimitives() { + final JsonSerialiser ioSerialiser = new JsonSerialiser(new FastByteBuffer(1000, true, null)); + + ioSerialiser.putHeaderInfo(); + + final SimpleClass simpleObj = new SimpleClass(); + simpleObj.setValues(); + final SimpleClass simpleObj2 = new SimpleClass(); + simpleObj2.foo = "baz"; + simpleObj2.integer = 42; + simpleObj2.switches = Collections.emptyList(); + ioSerialiser.put("SimpleObjects", List.of(simpleObj, simpleObj2), SimpleClass.class); + + ioSerialiser.putEndMarker(new WireDataFieldDescription(ioSerialiser, null, "end marker".hashCode(), "end marker", DataType.END_MARKER, -1, -1, -1)); + + ioSerialiser.getBuffer().flip(); + + final String result = new String(Arrays.copyOfRange(ioSerialiser.getBuffer().elements(), 0, ioSerialiser.getBuffer().limit())); + LOGGER.atDebug().addArgument(result).log("serialised:\n{}"); + + final Iterator lines = result.lines().iterator(); + assertEquals("{", lines.next()); + assertEquals(" \"SimpleObjects\": [{\"integer\":1337,\"foo\":\"bar\",\"switches\":[true,true,false]}, {\"integer\":42,\"foo\":\"baz\",\"switches\":[]}]", lines.next()); + assertEquals("}", lines.next()); + assertFalse(lines.hasNext()); + + // ensures that it is valid json + final WireDataFieldDescription header = ioSerialiser.parseIoStream(true); + + // reading back is not possible, because we cannot specify the type of the list and get a list of maps + ioSerialiser.setQueryFieldName("SimpleObjects", -1); + final List> recovered = ioSerialiser.getList(new ArrayList<>()); + assertThat(recovered, Matchers.contains(Matchers.aMapWithSize(3), Matchers.aMapWithSize(3))); + } + + @Test + @DisplayName("Simple object (de)serialisation") + void testSimpleObjectSerDe() { + final SimpleClass toSerialise = new SimpleClass(); + toSerialise.setValues(); + + final JsonSerialiser ioSerialiser = new JsonSerialiser(new FastByteBuffer(1000, true, null)); + + ioSerialiser.serialiseObject(toSerialise); + + ioSerialiser.getBuffer().flip(); + + final String result = new String(Arrays.copyOfRange(ioSerialiser.getBuffer().elements(), 0, ioSerialiser.getBuffer().limit())); + LOGGER.atDebug().addArgument(result).log("serialised:\n{}"); + + assertEquals("{\"integer\":1337,\"foo\":\"bar\",\"switches\":[true,true,false]}", result); + + final SimpleClass deserialized = ioSerialiser.deserialiseObject(new SimpleClass()); + assertEquals(toSerialise, deserialized); + } + + @Test + @DisplayName("Null object (de)serialisation") + void testNullObjectSerDe() { + final JsonSerialiser ioSerialiser = new JsonSerialiser(new FastByteBuffer(1000, true, null)); + + ioSerialiser.serialiseObject(null); + + ioSerialiser.getBuffer().flip(); + + final String result = new String(Arrays.copyOfRange(ioSerialiser.getBuffer().elements(), 0, ioSerialiser.getBuffer().limit())); + LOGGER.atDebug().addArgument(result).log("serialised:\n{}"); + + assertEquals("null", result); + + final SimpleClass deserialized = ioSerialiser.deserialiseObject(new SimpleClass()); + assertNull(deserialized); + } + + public static class SimpleClass { + public int integer = -1; + public String foo = null; + public List switches = null; + + public void setValues() { + integer = 1337; + foo = "bar"; + switches = List.of(true, true, false); + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (!(o instanceof SimpleClass)) + return false; + final SimpleClass that = (SimpleClass) o; + return integer == that.integer && Objects.equals(foo, that.foo) && Objects.equals(switches, that.switches); + } + + @Override + public int hashCode() { + return Objects.hash(integer, foo, switches); + } + } +} \ No newline at end of file diff --git a/serialiser/src/test/java/io/opencmw/serialiser/spi/helper/MyGenericClass.java b/serialiser/src/test/java/io/opencmw/serialiser/spi/helper/MyGenericClass.java new file mode 100644 index 00000000..337f3889 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/spi/helper/MyGenericClass.java @@ -0,0 +1,547 @@ +package io.opencmw.serialiser.spi.helper; + +import java.util.ArrayList; +import java.util.Arrays; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.spi.LoggingEventBuilder; + +import io.opencmw.serialiser.spi.BinarySerialiser; + +/** + * Generic test class to verify serialiser forward/backward identities. + * + * @author rstein + * @see BinarySerialiser + */ +@SuppressWarnings("ALL") +public class MyGenericClass { + private static final Logger LOGGER = LoggerFactory.getLogger(MyGenericClass.class); + private static boolean verboseLogging = false; + private static boolean extendedTestCase = true; + private static final String MODIFIED = "Modified"; + // supported data types + protected boolean dummyBoolean; + protected byte dummyByte; + protected short dummyShort; + protected int dummyInt; + protected long dummyLong; + protected float dummyFloat; + protected double dummyDouble; + protected String dummyString = "Test"; + + protected TestEnum enumState = TestEnum.TEST_CASE_1; + + protected ArrayList arrayListInteger = new ArrayList<>(); + protected ArrayList arrayListString = new ArrayList<>(); + + protected BoxedPrimitivesSubClass boxedPrimitivesNull; + public BoxedPrimitivesSubClass boxedPrimitives = new BoxedPrimitivesSubClass(); + + public ArraySubClass arrays = new ArraySubClass(); + public BoxedObjectArraySubClass objArrays = new BoxedObjectArraySubClass(); + + public MyGenericClass() { + arrayListInteger.add(1); + arrayListInteger.add(2); + arrayListInteger.add(3); + arrayListString.add("String#1"); + arrayListString.add("String#2"); + arrayListString.add("String#3"); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof MyGenericClass)) { + return false; + } + // normally this value is immediately returned for 'false', + // here: state is latched to detect potential other violations + boolean state = true; + MyGenericClass other = (MyGenericClass) obj; + if (this.hashCode() != other.hashCode()) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(this.hashCode()).addArgument(other.hashCode()).log("{} - hashCode is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyBoolean != other.dummyBoolean) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoolean).addArgument(other.dummyBoolean).log("{} - dummyBoolean is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyByte != other.dummyByte) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyByte).addArgument(other.dummyByte).log("{} - dummyByte is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyShort != other.dummyShort) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyShort).addArgument(other.dummyShort).log("{} - dummyShort is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyInt != other.dummyInt) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyInt).addArgument(other.dummyInt).log("{} - dummyInt is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyLong != other.dummyLong) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyLong).addArgument(other.dummyLong).log("{} - dummyLong is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyFloat != other.dummyFloat) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyFloat).addArgument(other.dummyFloat).log("{} - dummyFloat is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyDouble != other.dummyDouble) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyDouble).addArgument(other.dummyDouble).log("{} - dummyDouble is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (dummyString == null || !dummyString.contentEquals(other.dummyString)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyString).addArgument(other.dummyString).addArgument(dummyString).addArgument(other.dummyString).log("{} - dummyString is not equal {} vs {}'"); + state = false; + } + + if (!boxedPrimitives.equals(other.boxedPrimitives)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(boxedPrimitives).addArgument(other.boxedPrimitives).log("{} - boxedPrimitives are not equal: this '{}' vs. other '{}'"); + state = false; + } + + if (!extendedTestCase) { + // abort equals for more complex/extended data structures + return state; + } + + if (!enumState.equals(other.enumState)) { + state = false; + } + + if (!arrayListInteger.equals(other.arrayListInteger)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(arrayListInteger).addArgument(other.arrayListInteger).log("{} - arrayListInteger is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!arrayListString.equals(other.arrayListString)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(arrayListString).addArgument(other.arrayListString).log("{} - arrayListString is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!arrays.equals(other.arrays)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).log("{} - arrays are not equal"); + state = false; + } + if (!objArrays.equals(other.objArrays)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).log("{} - objArrays is not equal"); + state = false; + } + + return state; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + if (extendedTestCase) { + result = prime * result + ((enumState == null) ? 0 : enumState.hashCode()); + result = prime * result + ((arrayListInteger == null) ? 0 : arrayListInteger.hashCode()); + result = prime * result + ((arrayListString == null) ? 0 : arrayListString.hashCode()); + result = prime * result + ((arrays == null) ? 0 : arrays.hashCode()); + result = prime * result + ((boxedPrimitives == null) ? 0 : boxedPrimitives.hashCode()); + result = prime * result + ((boxedPrimitivesNull == null) ? 0 : boxedPrimitivesNull.hashCode()); + } + result = prime * result + (dummyBoolean ? 1231 : 1237); + result = prime * result + dummyByte; + long temp; + temp = Double.doubleToLongBits(dummyDouble); + result = prime * result + (int) (temp ^ (temp >>> 32)); + result = prime * result + Float.floatToIntBits(dummyFloat); + result = prime * result + dummyInt; + result = prime * result + (int) (dummyLong ^ (dummyLong >>> 32)); + result = prime * result + dummyShort; + result = prime * result + ((dummyString == null) ? 0 : dummyString.hashCode()); + // result = prime * result + ((objArrays == null) ? 0 : objArrays.hashCode()); + return result; + } + + public void modifyValues() { + dummyBoolean = !dummyBoolean; + dummyByte = (byte) (dummyByte + 1); + dummyShort = (short) (dummyShort + 2); + dummyInt = dummyInt + 1; + dummyLong = dummyLong + 1; + dummyFloat = dummyFloat + 0.5f; + dummyDouble = dummyDouble + 1.5; + dummyString = MODIFIED + dummyString; + arrayListInteger.set(0, arrayListInteger.get(0) + 1); + arrayListString.set(0, MODIFIED + arrayListString.get(0)); + enumState = TestEnum.values()[(enumState.ordinal() + 1) % TestEnum.values().length]; + } + + @Override + public String toString() { + return // + "[dummyBoolean=" + dummyBoolean + // + ", dummyByte=" + dummyByte + // + ", dummyShort=" + dummyShort + // + ", dummyInt=" + dummyInt + // + ", dummyFloat=" + dummyFloat + // + ", dummyDouble=" + dummyDouble + // + ", dummyString=" + dummyString + // + ", arrayListInteger=" + arrayListInteger + // + ", arrayListString=" + arrayListString + // + ", hash(boxedPrimitives)=" + boxedPrimitives.hashCode() + // + ", hash(arrays)=" + arrays.hashCode() + // + ", hash(objArrays)=" + objArrays.hashCode() + // + ']'; + } + + public static boolean isExtendedTestCase() { + return extendedTestCase; + } + + public static boolean isVerboseChecks() { + return verboseLogging; + } + + private static LoggingEventBuilder logBackEnd() { + return verboseLogging ? LOGGER.atError() : LOGGER.atTrace(); + } + + public static void setEnableExtendedTestCase(final boolean state) { + extendedTestCase = state; + } + + public static void setVerboseChecks(final boolean state) { + verboseLogging = state; + } + + public static class ArraySubClass { + protected boolean[] dummyBooleanArray = new boolean[2]; + protected byte[] dummyByteArray = new byte[2]; + protected short[] dummyShortArray = new short[2]; + protected char[] dummyCharArray = new char[2]; + protected int[] dummyIntArray = new int[2]; + protected long[] dummyLongArray = new long[2]; + protected float[] dummyFloatArray = new float[2]; + protected double[] dummyDoubleArray = new double[2]; + protected String[] dummyStringArray = { "Test 1", "Test 2" }; + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ArraySubClass)) { + return false; + } + // normally this value is immediately returned for 'false', + // here: state is latched to detect potential other violations + boolean state = true; + ArraySubClass other = (ArraySubClass) obj; + if (this.hashCode() != other.hashCode()) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(this.hashCode()).addArgument(other.hashCode()).log("{} - hashCode is not equal: this '{}' vs. other '{}'"); + state = false; + } + + if (!Arrays.equals(dummyBooleanArray, other.dummyBooleanArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBooleanArray).addArgument(other.dummyBooleanArray).log("{} - dummyBooleanArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyByteArray, other.dummyByteArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyByteArray).addArgument(other.dummyByteArray).log("{} - dummyByteArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyShortArray, other.dummyShortArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyShortArray).addArgument(other.dummyShortArray).log("{} - dummyShortArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyCharArray, other.dummyCharArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyCharArray).addArgument(other.dummyCharArray).log("{} - dummyCharArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyIntArray, other.dummyIntArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyIntArray).addArgument(other.dummyIntArray).log("{} - dummyIntArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyLongArray, other.dummyLongArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyLongArray).addArgument(other.dummyLongArray).log("{} - dummyLongArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyFloatArray, other.dummyFloatArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyFloatArray).addArgument(other.dummyFloatArray).log("{} - dummyFloatArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyDoubleArray, other.dummyDoubleArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyDoubleArray).addArgument(other.dummyDoubleArray).log("{} - dummyDoubleArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyStringArray, other.dummyStringArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyStringArray).addArgument(other.dummyStringArray).log("{} - dummyStringArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + + return state; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(dummyBooleanArray); + result = prime * result + Arrays.hashCode(dummyByteArray); + result = prime * result + Arrays.hashCode(dummyDoubleArray); + result = prime * result + Arrays.hashCode(dummyFloatArray); + result = prime * result + Arrays.hashCode(dummyIntArray); + result = prime * result + Arrays.hashCode(dummyLongArray); + result = prime * result + Arrays.hashCode(dummyShortArray); + result = prime * result + Arrays.hashCode(dummyCharArray); + result = prime * result + Arrays.hashCode(dummyStringArray); + return result; + } + + public void modifyValues() { + dummyBooleanArray[0] = !dummyBooleanArray[0]; + dummyByteArray[0] = (byte) (dummyByteArray[0] + (byte) 1); + dummyShortArray[0] = (short) (dummyShortArray[0] + (short) 1); + dummyCharArray[0] = (char) (dummyCharArray[0] + (char) 1); + dummyIntArray[0] = dummyIntArray[0] + 1; + dummyLongArray[0] = dummyLongArray[0] + 1L; + dummyFloatArray[0] = dummyFloatArray[0] + 0.5f; + dummyDoubleArray[0] = dummyDoubleArray[0] + 1.5; + dummyStringArray[0] = MODIFIED + dummyStringArray[0]; + } + } + + public static class BoxedObjectArraySubClass { + protected Boolean[] dummyBoxedBooleanArray = { false, false }; + protected Byte[] dummyBoxedByteArray = { 0, 0 }; + protected Short[] dummyBoxedShortArray = { 0, 0 }; + protected Character[] dummyBoxedCharArray = { 0, 0 }; + protected Integer[] dummyBoxedIntArray = { 0, 0 }; + protected Long[] dummyBoxedLongArray = { 0L, 0L }; + protected Float[] dummyBoxedFloatArray = { 0.0f, 0.0f }; + protected Double[] dummyBoxedDoubleArray = { 0.0, 0.0 }; + protected String[] dummyBoxedStringArray = { "TestString#2", "TestString#2" }; + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof BoxedObjectArraySubClass)) { + return false; + } + // normally this value is immediately returned for 'false', + // here: state is latched to detect potential other violations + boolean state = true; + BoxedObjectArraySubClass other = (BoxedObjectArraySubClass) obj; + if (this.hashCode() != other.hashCode()) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(this.hashCode()).addArgument(other.hashCode()).log("{} - hashCode is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedBooleanArray, other.dummyBoxedBooleanArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedBooleanArray).addArgument(other.dummyBoxedBooleanArray).log("{} - dummyBoxedBooleanArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedByteArray, other.dummyBoxedByteArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedByteArray).addArgument(other.dummyBoxedByteArray).log("{} - dummyBoxedByteArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedDoubleArray, other.dummyBoxedDoubleArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedDoubleArray).addArgument(other.dummyBoxedDoubleArray).log("{} - dummyBoxedDoubleArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedFloatArray, other.dummyBoxedFloatArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedFloatArray).addArgument(other.dummyBoxedFloatArray).log("{} - dummyBoxedFloatArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedIntArray, other.dummyBoxedIntArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedIntArray).addArgument(other.dummyBoxedIntArray).log("{} - dummyBoxedIntArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedLongArray, other.dummyBoxedLongArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedLongArray).addArgument(other.dummyBoxedLongArray).log("{} - dummyBoxedLongArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedShortArray, other.dummyBoxedShortArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedShortArray).addArgument(other.dummyBoxedShortArray).log("{} - dummyBoxedShortArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedCharArray, other.dummyBoxedCharArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedCharArray).addArgument(other.dummyBoxedCharArray).log("{} - dummyBoxedCharArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!Arrays.equals(dummyBoxedStringArray, other.dummyBoxedStringArray)) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedStringArray).addArgument(other.dummyBoxedStringArray).log("{} - dummyBoxedStringArray is not equal: this '{}' vs. other '{}'"); + state = false; + } + + return state; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(dummyBoxedBooleanArray); + result = prime * result + Arrays.hashCode(dummyBoxedByteArray); + result = prime * result + Arrays.hashCode(dummyBoxedDoubleArray); + result = prime * result + Arrays.hashCode(dummyBoxedFloatArray); + result = prime * result + Arrays.hashCode(dummyBoxedIntArray); + result = prime * result + Arrays.hashCode(dummyBoxedLongArray); + result = prime * result + Arrays.hashCode(dummyBoxedShortArray); + result = prime * result + Arrays.hashCode(dummyBoxedStringArray); + return result; + } + + public void modifyValues() { + dummyBoxedBooleanArray[0] = !dummyBoxedBooleanArray[0]; + dummyBoxedByteArray[0] = (byte) (dummyBoxedByteArray[0] + (byte) 1); + dummyBoxedShortArray[0] = (short) (dummyBoxedShortArray[0] + (short) 1); + dummyBoxedCharArray[0] = (char) (dummyBoxedCharArray[0] + (char) 1); + dummyBoxedIntArray[0] = dummyBoxedIntArray[0] + 1; + dummyBoxedLongArray[0] = dummyBoxedLongArray[0] + 1L; + dummyBoxedFloatArray[0] = dummyBoxedFloatArray[0] + 0.5f; + dummyBoxedDoubleArray[0] = dummyBoxedDoubleArray[0] + 1.5; + dummyBoxedStringArray[0] = MODIFIED + dummyBoxedStringArray[0]; + } + } + + public class BoxedPrimitivesSubClass { + protected Boolean dummyBoxedBoolean = Boolean.FALSE; + protected Byte dummyBoxedByte = (byte) 0; + protected Short dummyBoxedShort = (short) 0; + protected Integer dummyBoxedInt = 0; + protected Long dummyBoxedLong = 0L; + protected Float dummyBoxedFloat = 0f; + protected Double dummyBoxedDouble = 0.0; + protected String dummyBoxedString = "Test"; + + protected BoxedPrimitivesSubSubClass boxedPrimitivesSubSubClass = new BoxedPrimitivesSubSubClass(); + protected BoxedPrimitivesSubSubClass boxedPrimitivesSubSubClassNull; // to check instantiation + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof BoxedPrimitivesSubClass)) { + return false; + } + // normally this value is immediately returned for 'false', + // here: state is latched to detect potential other violations + boolean state = true; + BoxedPrimitivesSubClass other = (BoxedPrimitivesSubClass) obj; + if (this.hashCode() != other.hashCode()) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(this.hashCode()).addArgument(other.hashCode()).log("{} - hashCode is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedBoolean == null && other.dummyBoxedBoolean == null) && (dummyBoxedBoolean == null || other.dummyBoxedBoolean == null || !dummyBoxedBoolean.equals(other.dummyBoxedBoolean))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedBoolean).addArgument(other.dummyBoxedBoolean).log("{} - dummyBoxedBoolean is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedByte == null && other.dummyBoxedByte == null) && (dummyBoxedByte == null || other.dummyBoxedByte == null || !dummyBoxedByte.equals(other.dummyBoxedByte))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedByte).addArgument(other.dummyBoxedByte).log("{} - dummyBoxedByte is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedShort == null && other.dummyBoxedShort == null) && (dummyBoxedShort == null || other.dummyBoxedShort == null || !dummyBoxedShort.equals(other.dummyBoxedShort))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedShort).addArgument(other.dummyBoxedShort).log("{} - dummyBoxedShort is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedInt == null && other.dummyBoxedInt == null) && (dummyBoxedInt == null || other.dummyBoxedInt == null || !dummyBoxedInt.equals(other.dummyBoxedInt))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedInt).addArgument(other.dummyBoxedInt).log("{} - dummyBoxedInt is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedLong == null && other.dummyBoxedLong == null) && (dummyBoxedLong == null || other.dummyBoxedLong == null || !dummyBoxedLong.equals(other.dummyBoxedLong))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedLong).addArgument(other.dummyBoxedLong).log("{} - dummyBoxedLong is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedFloat == null && other.dummyBoxedFloat == null) && (dummyBoxedFloat == null || other.dummyBoxedFloat == null || !dummyBoxedFloat.equals(other.dummyBoxedFloat))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedFloat).addArgument(other.dummyBoxedFloat).log("{} - dummyBoxedFloat is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedDouble == null && other.dummyBoxedDouble == null) && (dummyBoxedDouble == null || other.dummyBoxedDouble == null || !dummyBoxedDouble.equals(other.dummyBoxedDouble))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedDouble).addArgument(other.dummyBoxedDouble).log("{} - dummyBoxedDouble is not equal: this '{}' vs. other '{}'"); + state = false; + } + if (!(dummyBoxedString == null && other.dummyBoxedString == null) && (dummyBoxedString == null || other.dummyBoxedString == null || !dummyBoxedString.equals(other.dummyBoxedString))) { + logBackEnd().addArgument(this.getClass().getSimpleName()).addArgument(dummyBoxedString).addArgument(other.dummyBoxedString).log("{} - dummyBoxedString is not equal: this '{}' vs. other '{}'"); + state = false; + } + + return state; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((dummyBoxedBoolean == null) ? 0 : dummyBoxedBoolean.hashCode()); + result = prime * result + ((dummyBoxedByte == null) ? 0 : dummyBoxedByte.hashCode()); + result = prime * result + ((dummyBoxedDouble == null) ? 0 : dummyBoxedDouble.hashCode()); + result = prime * result + ((dummyBoxedFloat == null) ? 0 : dummyBoxedFloat.hashCode()); + result = prime * result + ((dummyBoxedInt == null) ? 0 : dummyBoxedInt.hashCode()); + result = prime * result + ((dummyBoxedLong == null) ? 0 : dummyBoxedLong.hashCode()); + result = prime * result + ((dummyBoxedShort == null) ? 0 : dummyBoxedShort.hashCode()); + result = prime * result + ((dummyBoxedString == null) ? 0 : dummyBoxedString.hashCode()); + return result; + } + + public void modifyValues() { + dummyBoxedBoolean = !dummyBoxedBoolean; + dummyBoxedByte = (byte) (dummyBoxedByte + 1); + dummyBoxedShort = (short) (dummyBoxedShort + 2); + dummyBoxedInt = dummyBoxedInt + 1; + dummyBoxedLong = dummyBoxedLong + 1; + dummyBoxedFloat = dummyBoxedFloat + 0.5f; + dummyBoxedDouble = dummyBoxedDouble + 1.5; + dummyBoxedString = MODIFIED + dummyBoxedString; + arrayListInteger.set(0, arrayListInteger.get(0) + 1); + arrayListString.set(0, MODIFIED + arrayListString.get(0)); + } + + @Override + public String toString() { + return // + "[dummyBoxedBoolean=" + dummyBoxedBoolean + // + ", dummyBoxedByte=" + dummyBoxedByte + // + ", dummyBoxedShort=" + dummyBoxedShort + // + ", dummyBoxedInt=" + dummyBoxedInt + // + ", dummyBoxedFloat=" + dummyBoxedFloat + // + ", dummyBoxedDouble=" + dummyBoxedDouble + // + ", dummyBoxedString=" + dummyBoxedString + // + ']'; + } + + @SuppressWarnings("hiding") + public class BoxedPrimitivesSubSubClass { + protected Boolean dummyBoxedBooleanL2 = true; + protected Byte dummyBoxedByteL2 = (byte) 0; + protected Short dummyBoxedShortL2 = (short) 0; + protected Integer dummyBoxedIntL2 = 0; + protected Long dummyBoxedLongL2 = 0L; + protected Float dummyBoxedFloatL2 = 0f; + protected Double dummyBoxedDoubleL2 = 0.0; + protected String dummyBoxedStringL2 = "Test"; + + public BoxedPrimitivesSubSubSubClass boxedPrimitivesSubSubSubClass = new BoxedPrimitivesSubSubSubClass(); + public BoxedPrimitivesSubSubSubClass boxedPrimitivesSubSubSubClassNull; // to check instantiation + + @SuppressWarnings("hiding") + public class BoxedPrimitivesSubSubSubClass { + protected Boolean dummyBoxedBooleanL3 = false; + protected Byte dummyBoxedByteL3 = (byte) 0; + protected Short dummyBoxedShortL3 = (short) 0; + protected Integer dummyBoxedIntL3 = 0; + protected Long dummyBoxedLongL3 = 0L; + protected Float dummyBoxedFloatL3 = 0f; + protected Double dummyBoxedDoubleL3 = 0.0; + protected String dummyBoxedStringL3 = "Test"; + } + } + } + + public enum TestEnum { + TEST_CASE_1, + TEST_CASE_2, + TEST_CASE_3, + TEST_CASE_4 + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/spi/iobuffer/DataSetSerialiserTests.java b/serialiser/src/test/java/io/opencmw/serialiser/spi/iobuffer/DataSetSerialiserTests.java new file mode 100644 index 00000000..686f866d --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/spi/iobuffer/DataSetSerialiserTests.java @@ -0,0 +1,433 @@ +package io.opencmw.serialiser.spi.iobuffer; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.stream.Stream; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ByteBuffer; +import io.opencmw.serialiser.spi.FastByteBuffer; + +import de.gsi.dataset.DataSet; +import de.gsi.dataset.DataSet2D; +import de.gsi.dataset.DataSetError; +import de.gsi.dataset.DataSetMetaData; +import de.gsi.dataset.event.EventListener; +import de.gsi.dataset.spi.AbstractDataSet; +import de.gsi.dataset.spi.DefaultErrorDataSet; +import de.gsi.dataset.spi.DoubleDataSet; +import de.gsi.dataset.spi.DoubleErrorDataSet; +import de.gsi.dataset.spi.DoubleGridDataSet; +import de.gsi.dataset.testdata.spi.TriangleFunction; + +/** + * @author Alexander Krimm + * @author rstein + */ +class DataSetSerialiserTests { + private static final int BUFFER_SIZE = 10000; + private static final String[] DEFAULT_AXES_NAME = { "x", "y", "z" }; + private static final double DELTA = 1e-3; + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testDataSet(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + boolean asFloat32 = false; + final DoubleDataSet original = new DoubleDataSet(new TriangleFunction("test", 1009)); + addMetaData(original, true); + + final DataSetSerialiser ioSerialiser = DataSetSerialiser.withIoSerialiser(new BinarySerialiser(buffer)); + + ioSerialiser.write(original, false); + buffer.reset(); // reset to read position (==0) + final DataSet restored = ioSerialiser.read(); + + assertEquals(original, restored); + } + + @ParameterizedTest(name = "IoBuffer class - {0}, asFloat - {1}") + @MethodSource("buffersAndFloatParameters") + void testGridDataSet(final Class bufferClass, final boolean asFloat32) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + final DoubleGridDataSet original = asFloat32 ? new DoubleGridDataSet("test", false, + new double[][] { { 1f, 2f }, { 0.1f, 0.2f, 0.3f } }, new double[] { 9.9f, 8.8f, 7.7f, 6.6f, 5.5f, 4.4f }) + : new DoubleGridDataSet("test", false, + new double[][] { { 1.0, 2.0 }, { 0.1, 0.2, 0.3 } }, new double[] { 9.9, 8.8, 7.7, 6.6, 5.5, 4.4 }); + + final DataSetSerialiser ioSerialiser = DataSetSerialiser.withIoSerialiser(new BinarySerialiser(buffer)); + + ioSerialiser.write(original, asFloat32); + buffer.reset(); // reset to read position (==0) + final DataSet restored = ioSerialiser.read(); + + assertEquals(original, restored); + } + static public Stream buffersAndFloatParameters() { + return Stream.of( + Arguments.arguments(ByteBuffer.class, true), + Arguments.arguments(ByteBuffer.class, false), + Arguments.arguments(FastByteBuffer.class, true), + Arguments.arguments(FastByteBuffer.class, false)); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testDataSetErrorSymmetric(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(BUFFER_SIZE); + boolean asFloat32 = false; + + final DefaultErrorDataSet original = new DefaultErrorDataSet("test", new double[] { 1, 2, 3 }, + new double[] { 6, 7, 8 }, new double[] { 7, 8, 9 }, new double[] { 7, 8, 9 }, 3, false) { + private static final long serialVersionUID = 1L; + + @Override + public ErrorType getErrorType(int dimIndex) { + if (dimIndex == 1) { + return ErrorType.SYMMETRIC; + } + return super.getErrorType(dimIndex); + } + }; + addMetaData(original, true); + + final DataSetSerialiser ioSerialiser = DataSetSerialiser.withIoSerialiser(new BinarySerialiser(buffer)); + ioSerialiser.write(original, false); + buffer.reset(); // reset to read position (==0) + final DefaultErrorDataSet restored = (DefaultErrorDataSet) ioSerialiser.read(); + + assertEquals(new DefaultErrorDataSet(original), new DefaultErrorDataSet(restored)); + } + + @DisplayName("test getDoubleArray([boolean[], byte[], ..., String[]) helper method") + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testGetDoubleArrayHelper(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); // a bit larger buffer since we test more cases at once + final BinarySerialiser ioSerialiser = new BinarySerialiser(buffer); + + putGenericTestArrays(ioSerialiser); + + buffer.reset(); + + // test conversion to double array + ioSerialiser.checkHeaderInfo(); + assertThrows(IllegalArgumentException.class, () -> DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.OTHER)); + assertArrayEquals(new double[] { 1.0, 0.0, 1.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.BOOL_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.BYTE_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.CHAR_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.SHORT_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.INT_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.LONG_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.FLOAT_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.DOUBLE_ARRAY)); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, DataSetSerialiser.getDoubleArray(ioSerialiser, null, DataType.STRING_ARRAY)); + } + + private static void putGenericTestArrays(final BinarySerialiser ioSerialiser) { + ioSerialiser.putHeaderInfo(); + ioSerialiser.putGenericArrayAsPrimitive(DataType.BOOL, new Boolean[] { true, false, true }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.BYTE, new Byte[] { (byte) 1, (byte) 0, (byte) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.CHAR, new Character[] { (char) 1, (char) 0, (char) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.SHORT, new Short[] { (short) 1, (short) 0, (short) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.INT, new Integer[] { 1, 0, 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.LONG, new Long[] { 1L, 0L, 2L }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.FLOAT, new Float[] { (float) 1, (float) 0, (float) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.DOUBLE, new Double[] { (double) 1, (double) 0, (double) 2 }, 3); + ioSerialiser.putGenericArrayAsPrimitive(DataType.STRING, new String[] { "1.0", "0.0", "2.0" }, 3); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testDataSetFloatError(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + boolean asFloat32 = true; + final DefaultErrorDataSet original = new DefaultErrorDataSet("test", new double[] { 1f, 2f, 3f }, + new double[] { 6f, 7f, 8f }, new double[] { 7f, 8f, 9f }, new double[] { 7f, 8f, 9f }, 3, false); + addMetaData(original, true); + + final DataSetSerialiser ioSerialiser = DataSetSerialiser.withIoSerialiser(new BinarySerialiser(buffer)); + + ioSerialiser.write(original, true); + buffer.reset(); // reset to read position (==0) + final DataSet restored = ioSerialiser.read(); + + assertEquals(original, restored); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testDataSetFloatErrorSymmetric(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + boolean asFloat32 = true; + + final DefaultErrorDataSet original = new DefaultErrorDataSet("test", new double[] { 1f, 2f, 3f }, + new double[] { 6f, 7f, 8f }, new double[] { 7f, 8f, 9f }, new double[] { 7f, 8f, 9f }, 3, false) { + private static final long serialVersionUID = 1L; + + @Override + public ErrorType getErrorType(int dimIndex) { + if (dimIndex == 1) { + return ErrorType.SYMMETRIC; + } + return super.getErrorType(dimIndex); + } + }; + addMetaData(original, true); + + final DataSetSerialiser ioSerialiser = DataSetSerialiser.withIoSerialiser(new BinarySerialiser(buffer)); + + ioSerialiser.write(original, true); + buffer.reset(); // reset to read position (==0) + final DefaultErrorDataSet restored = (DefaultErrorDataSet) ioSerialiser.read(); + + assertEquals(new DefaultErrorDataSet(original), new DefaultErrorDataSet(restored)); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testErrorDataSet(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(10 * BUFFER_SIZE); + + boolean asFloat32 = false; + final DoubleErrorDataSet original = new DoubleErrorDataSet(new TriangleFunction("test", 1009)); + addMetaData(original, true); + + final DataSetSerialiser ioSerialiser = DataSetSerialiser.withIoSerialiser(new BinarySerialiser(buffer)); + + ioSerialiser.write(original, false); + buffer.reset(); // reset to read position (==0) + final DataSet restored = ioSerialiser.read(); + + assertEquals(original, restored); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testGenericSerialiserIdentity(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + IoClassSerialiser serialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + + final DefaultErrorDataSet original = new DefaultErrorDataSet("test", // + new double[] { 1f, 2f, 3f }, new double[] { 6f, 7f, 8f }, // + new double[] { 0.7f, 0.8f, 0.9f }, new double[] { 7f, 8f, 9f }, 3, false); + addMetaData(original, true); + final EventListener eventListener = evt -> { + // empty eventLister for counting + }; + original.addListener(eventListener); + assertEquals(1, original.updateEventListener().size()); + DataSetWrapper dsOrig = new DataSetWrapper(); + dsOrig.source = original; + DataSetWrapper cpOrig = new DataSetWrapper(); + + // serialise-deserialise DataSet + buffer.reset(); // '0' writing at start of buffer + serialiser.serialiseObject(dsOrig); + + // buffer.reset(); // reset to read position (==0) + // final WireDataFieldDescription root = serialiser.getIoSerialiser().parseIoStream(true); + // root.printFieldStructure(); + + buffer.reset(); // reset to read position (==0) + final Object retOrig = serialiser.deserialiseObject(cpOrig); + + assertSame(cpOrig, retOrig, "Deserialisation expected to be in-place"); + + // check DataSet for equality + if (!(cpOrig.source instanceof DataSetError)) { + throw new IllegalStateException("DataSet '" + cpOrig.source + "' is not not instanceof DataSetError"); + } + assertEquals(0, cpOrig.source.updateEventListener().size()); + DataSetError test = (DataSetError) (cpOrig.source); + + testIdentityCore(original, test); + testIdentityLabelsAndStyles(original, test, true); + if (test instanceof DataSetMetaData) { + testIdentityMetaData(original, (DataSetMetaData) test, true); + } + assertEquals(dsOrig.source, test); + } + + @ParameterizedTest(name = "IoBuffer class - {0}") + @ValueSource(classes = { ByteBuffer.class, FastByteBuffer.class }) + void testGenericSerialiserInplaceIdentity(final Class bufferClass) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { + assertNotNull(bufferClass, "bufferClass being not null"); + assertNotNull(bufferClass.getConstructor(int.class), "Constructor(Integer) present"); + final IoBuffer buffer = bufferClass.getConstructor(int.class).newInstance(2 * BUFFER_SIZE); + + IoClassSerialiser serialiser = new IoClassSerialiser(buffer, BinarySerialiser.class); + + final DefaultErrorDataSet original = new DefaultErrorDataSet("test", // + new double[] { 1f, 2f, 3f }, new double[] { 6f, 7f, 8f }, // + new double[] { 0.7f, 0.8f, 0.9f }, new double[] { 7f, 8f, 9f }, 3, false); + addMetaData(original, true); + final EventListener eventListener = evt -> { + // empty eventLister for counting + }; + original.addListener(eventListener); + assertEquals(1, original.updateEventListener().size()); + DataSetWrapper dsOrig = new DataSetWrapper(); + dsOrig.source = original; + DataSetWrapper cpOrig = new DataSetWrapper(); + cpOrig.source = new DefaultErrorDataSet("copyName - to be overwritten"); + final EventListener eventListener1 = evt -> { + // empty eventLister for counting + }; + cpOrig.source.addListener(eventListener1); + final EventListener eventListener2 = evt -> { + // empty eventLister for counting + }; + cpOrig.source.addListener(eventListener2); + assertEquals(2, cpOrig.source.updateEventListener().size()); + + // serialise-deserialise DataSet + buffer.reset(); // '0' writing at start of buffer + serialiser.serialiseObject(dsOrig); + + // buffer.reset(); // reset to read position (==0) + // final WireDataFieldDescription root = serialiser.getIoSerialiser().parseIoStream(true); + // root.printFieldStructure(); + + buffer.reset(); // reset to read position (==0) + final Object retOrig = serialiser.deserialiseObject(cpOrig); + + assertSame(cpOrig, retOrig, "Deserialisation expected to be in-place"); + + // check DataSet for equality + if (!(cpOrig.source instanceof DataSetError)) { + throw new IllegalStateException("DataSet '" + cpOrig.source + "' is not not instanceof DataSetError"); + } + assertEquals(2, cpOrig.source.updateEventListener().size()); + assertTrue(cpOrig.source.updateEventListener().contains(eventListener1)); + assertTrue(cpOrig.source.updateEventListener().contains(eventListener2)); + DataSetError test = (DataSetError) (cpOrig.source); + + testIdentityCore(original, test); + testIdentityLabelsAndStyles(original, test, true); + if (test instanceof DataSetMetaData) { + testIdentityMetaData(original, (DataSetMetaData) test, true); + } + + assertEquals(dsOrig.source, test); + } + + @Test + void testMiscellaneous() { + assertEquals(0, DataSetSerialiser.getDimIndex("axis0", "axis")); + assertDoesNotThrow(() -> DataSetSerialiser.getDimIndex("axi0", "axis")); + assertEquals(-1, DataSetSerialiser.getDimIndex("axi0", "axis")); + assertDoesNotThrow(() -> DataSetSerialiser.getDimIndex("axis0.1", "axis")); + assertEquals(-1, DataSetSerialiser.getDimIndex("axis0.1", "axis")); + } + + private static void addMetaData(final AbstractDataSet dataSet, final boolean addLabelsStyles) { + if (addLabelsStyles) { + dataSet.addDataLabel(1, "test"); + dataSet.addDataStyle(2, "color: red"); + } + dataSet.getMetaInfo().put("Test", "Value"); + dataSet.getErrorList().add("TestError"); + dataSet.getWarningList().add("TestWarning"); + dataSet.getInfoList().add("TestInfo"); + } + + private static String encodingBinary(final boolean isBinaryEncoding) { + return isBinaryEncoding ? "binary-based" : "string-based"; + } + + private static boolean floatInequality(double a, double b) { + // 32-bit float uses 23-bit for the mantissa + return Math.abs((float) a - (float) b) > 2 / Math.pow(2, 23); + } + + private static void testIdentityCore(final DataSetError original, final DataSetError test) { + // some checks + assertEquals(original.getName(), test.getName(), "name"); + assertEquals(original.getDimension(), test.getDimension(), "dimension"); + + assertEquals(original.getDataCount(), test.getDataCount(), "getDataCount()"); + + // check for numeric value + final int dataCount = original.getDataCount(); + for (int dim = 0; dim < original.getDimension(); dim++) { + final String dStr = dim < DEFAULT_AXES_NAME.length ? DEFAULT_AXES_NAME[dim] : "dim" + (dim + 1) + "-Axis"; + + assertEquals(original.getErrorType(dim), test.getErrorType(dim), dStr + " error Type"); + assertArrayEquals(Arrays.copyOfRange(original.getValues(dim), 0, dataCount), Arrays.copyOfRange(test.getValues(dim), 0, dataCount), DELTA, dStr + "-Values"); + assertArrayEquals(Arrays.copyOfRange(original.getErrorsPositive(dim), 0, dataCount), Arrays.copyOfRange(test.getErrorsPositive(dim), 0, dataCount), DELTA, dStr + "-Errors positive"); + assertArrayEquals(Arrays.copyOfRange(original.getErrorsNegative(dim), 0, dataCount), Arrays.copyOfRange(test.getErrorsNegative(dim), 0, dataCount), DELTA, dStr + "-Errors negative"); + } + } + + private static void testIdentityLabelsAndStyles(final DataSet2D originalDS, final DataSet testDS, final boolean binary) { + // check for labels & styles + for (int i = 0; i < originalDS.getDataCount(); i++) { + if (originalDS.getDataLabel(i) == null && testDS.getDataLabel(i) == null) { + // cannot compare null vs null + continue; + } + if (!originalDS.getDataLabel(i).equals(testDS.getDataLabel(i))) { + String msg = String.format("data set label do not match (%s): original(%d) ='%s' vs. copy(%d) ='%s' %n", + encodingBinary(binary), i, originalDS.getDataLabel(i), i, testDS.getDataLabel(i)); + throw new IllegalStateException(msg); + } + } + for (int i = 0; i < originalDS.getDataCount(); i++) { + if (originalDS.getStyle(i) == null && testDS.getStyle(i) == null) { + // cannot compare null vs null + continue; + } + if (!originalDS.getStyle(i).equals(testDS.getStyle(i))) { + String msg = String.format("data set style do not match (%s): original(%d) ='%s' vs. copy(%d) ='%s' %n", + encodingBinary(binary), i, originalDS.getStyle(i), i, testDS.getStyle(i)); + throw new IllegalStateException(msg); + } + } + } + + private static void testIdentityMetaData(final DataSetMetaData originalDS, final DataSetMetaData testDS, final boolean binary) { + // check for meta data and meta messages + if (!originalDS.getInfoList().equals(testDS.getInfoList())) { + String msg = String.format("data set info lists do not match (%s): original ='%s' vs. copy ='%s' %n", + encodingBinary(binary), originalDS.getInfoList(), testDS.getInfoList()); + throw new IllegalStateException(msg); + } + } + + private static class DataSetWrapper { + public DataSet source; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/CmwHelper.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/CmwHelper.java new file mode 100644 index 00000000..29433540 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/CmwHelper.java @@ -0,0 +1,181 @@ +package io.opencmw.serialiser.utils; + +public final class CmwHelper { + /* + private static final Logger LOGGER = LoggerFactory.getLogger(SerialiserQuickBenchmark.class); // N.B. SerialiserQuickBenchmark reference on purpose + private static final DataSerializer cmwSerializer = DataFactory.createDataSerializer(); + + public static Data getCmwData(final TestDataCl + jo) { + Data data = DataFactory.createData(); + + data.append("bool1", pojo.bool1); + data.append("bool2", pojo.bool2); + data.append("byte1", pojo.byte1); + data.append("byte2", pojo.byte2); + data.append("char1", (short) pojo.char1); // work-around storing char as short + data.append("char2", (short) pojo.char2); // work-around storing char as short + data.append("short1", pojo.short1); + data.append("short2", pojo.short2); + data.append("int1", pojo.int1); + data.append("int2", pojo.int2); + data.append("long1", pojo.long1); + data.append("long2", pojo.long2); + data.append("float1", pojo.float1); + data.append("float2", pojo.float2); + data.append("double1", pojo.double1); + data.append("double2", pojo.double2); + data.append("string1", pojo.string1); + // data.append("string2", pojo.string2); // N.B. CMW handles only ASCII characters + + // 1-dim array + data.appendArray("boolArray", pojo.boolArray); + data.appendArray("byteArray", pojo.byteArray); + //data.appendArray("charArray", pojo.charArray); // not supported by CMW + data.appendArray("shortArray", pojo.shortArray); + data.appendArray("intArray", pojo.intArray); + data.appendArray("longArray", pojo.longArray); + data.appendArray("floatArray", pojo.floatArray); + data.appendArray("doubleArray", pojo.doubleArray); + data.appendArray("stringArray", pojo.stringArray); + + // multidim arrays + data.appendArray("nDimensions", pojo.nDimensions); + data.appendMultiArray("boolNdimArray", pojo.boolNdimArray, pojo.nDimensions); + data.appendMultiArray("byteNdimArray", pojo.byteNdimArray, pojo.nDimensions); + //data.appendMultiArray("charNdimArray", pojo.charNdimArray, pojo.nDimensions); // not supported by CMW + data.appendMultiArray("shortNdimArray", pojo.shortNdimArray, pojo.nDimensions); + data.appendMultiArray("intNdimArray", pojo.intNdimArray, pojo.nDimensions); + data.appendMultiArray("longNdimArray", pojo.longNdimArray, pojo.nDimensions); + data.appendMultiArray("floatNdimArray", pojo.floatNdimArray, pojo.nDimensions); + data.appendMultiArray("doubleNdimArray", pojo.doubleNdimArray, pojo.nDimensions); + + if (pojo.nestedData != null) { + data.append("nestedData", getCmwData(pojo.nestedData)); + } + + return data; + } + + public static void applyCmwData(final Data data, final TestDataClass pojo) { + pojo.bool1 = data.getBool("bool1"); + pojo.bool2 = data.getBool("bool2"); + pojo.byte1 = data.getByte("byte1"); + pojo.byte2 = data.getByte("byte2"); + // pojo.char1 = data.getCharacter("char1"); // not supported by CMW + // pojo.char2 = data.getCharacter("char2"); // not supported by CMW + pojo.char1 = (char) data.getShort("char1"); // work-around + pojo.char2 = (char) data.getShort("char2"); // work-around + pojo.short1 = data.getShort("short1"); + pojo.short2 = data.getShort("short2"); + pojo.int1 = data.getInt("int1"); + pojo.int2 = data.getInt("int2"); + pojo.long1 = data.getLong("long1"); + pojo.long2 = data.getLong("long2"); + pojo.float1 = data.getFloat("float1"); + pojo.float2 = data.getFloat("float2"); + pojo.double1 = data.getDouble("double1"); + pojo.double2 = data.getDouble("double2"); + pojo.string1 = data.getString("string1"); + //pojo.string2 = data.getString("string2"); // N.B. handles only ASCII characters + + // 1-dim array + pojo.boolArray = data.getBoolArray("boolArray"); + pojo.byteArray = data.getByteArray("byteArray"); + // pojo.charArray = data.getCharacterArray("byteArray"); // not supported by CMW + pojo.shortArray = data.getShortArray("shortArray"); + pojo.intArray = data.getIntArray("intArray"); + pojo.longArray = data.getLongArray("longArray"); + pojo.floatArray = data.getFloatArray("floatArray"); + pojo.doubleArray = data.getDoubleArray("doubleArray"); + pojo.stringArray = data.getStringArray("stringArray"); + + // multi-dim arrays + pojo.nDimensions = data.getIntArray("nDimensions"); + pojo.boolNdimArray = data.getBoolMultiArray("boolNdimArray").getElements(); + pojo.byteNdimArray = data.getByteMultiArray("byteNdimArray").getElements(); + // pojo.charNdimArray = data.getCharMultiArray("byteArray"); // not supported by CMW + pojo.shortNdimArray = data.getShortMultiArray("shortNdimArray").getElements(); + pojo.intNdimArray = data.getIntMultiArray("intNdimArray").getElements(); + pojo.longNdimArray = data.getLongMultiArray("longNdimArray").getElements(); + pojo.floatNdimArray = data.getFloatMultiArray("floatNdimArray").getElements(); + pojo.doubleNdimArray = data.getDoubleMultiArray("doubleNdimArray").getElements(); + + final Entry nestedEntry = data.getEntry("nestedData"); + if (nestedEntry != null) { + if (pojo.nestedData == null) { + pojo.nestedData = new TestDataClass(-1, -1, -1); + } + applyCmwData(nestedEntry.getData(), pojo.nestedData); + } + } + + public static void testSerialiserPerformanceMap(final int iterations, final TestDataClass inputObject, final TestDataClass outputObject) { + final Data sourceData = CmwHelper.getCmwData(inputObject); + + final long startTime = System.nanoTime(); + byte[] buffer = new byte[0]; + for (int i = 0; i < iterations; i++) { + buffer = cmwSerializer.serializeToBinary(sourceData); + final Data retrievedData = cmwSerializer.deserializeFromBinary(buffer); + if (sourceData.size() != retrievedData.size()) { + // check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((buffer.length / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) buffer.length, true)) + .addArgument(diffMillis) // + .log("CMW Serializer (Map only) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void testPerformancePojo(final int iterations, final TestDataClass inputObject, final TestDataClass outputObject) { + final long startTime = System.nanoTime(); + + byte[] buffer = new byte[0]; + for (int i = 0; i < iterations; i++) { + buffer = cmwSerializer.serializeToBinary(CmwHelper.getCmwData(inputObject)); + final Data retrievedData = cmwSerializer.deserializeFromBinary(buffer); + CmwHelper.applyCmwData(retrievedData, outputObject); + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((buffer.length / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) buffer.length, true)) + .addArgument(diffMillis) // + .log("CMW Serializer (POJO) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void checkSerialiserIdentity(final TestDataClass inputObject, final TestDataClass outputObject) { + outputObject.clear(); + final Data sourceData = getCmwData(inputObject); + final byte[] buffer = cmwSerializer.serializeToBinary(sourceData); + final Data retrievedData = cmwSerializer.deserializeFromBinary(buffer); + applyCmwData(retrievedData, outputObject); + final int nBytes = buffer.length; + LOGGER.atInfo().addArgument(nBytes).log("CMW serialiser nBytes = {}"); + + // disabled since UTF-8 is not supported which would fail this test for 'string2' which contains UTF-8 characters + //assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + } + + */ +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/CmwLightHelper.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/CmwLightHelper.java new file mode 100644 index 00000000..aaf8af30 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/CmwLightHelper.java @@ -0,0 +1,461 @@ +package io.opencmw.serialiser.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.benchmark.SerialiserQuickBenchmark; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.ProtocolInfo; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +public class CmwLightHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(SerialiserQuickBenchmark.class); // N.B. SerialiserQuickBenchmark reference on purpose + private static final IoBuffer byteBuffer = new FastByteBuffer(100000); + // private static final IoBuffer byteBuffer = new ByteBuffer(20000); + private static final CmwLightSerialiser cmwLightSerialiser = new CmwLightSerialiser(byteBuffer); + private static final IoClassSerialiser ioSerialiser = new IoClassSerialiser(byteBuffer, CmwLightSerialiser.class); + private static int nEntries = -1; + /* + public static void checkCmwLightVsCmwIdentityBackward(final TestDataClass inputObject, TestDataClass outputObject) { + final DataSerializer cmwSerializer = DataFactory.createDataSerializer(); + TestDataClass.setCmwCompatibilityMode(true); + + outputObject.clear(); + byteBuffer.reset(); + CmwLightHelper.serialiseCustom(cmwLightSerialiser, inputObject); + final int nBytesCmwLight = byteBuffer.position(); + LOGGER.atInfo().addArgument(nBytesCmwLight).log("backward compatibility check: CmwLight serialiser nBytes = {}"); + + // keep: checks serialised data structure + // wrapCmwBuffer.reset(); + // final WireDataFieldDescription fieldRoot = CmwLightHelper.deserialiseMap(cmwLightSerialiser); + // fieldRoot.printFieldStructure(); + + // N.B. cannot use custom deserialiser since entry order seems to be arbitrary in CMW Data object + byteBuffer.reset(); + final Data retrievedData = cmwSerializer.deserializeFromBinary(((FastByteBuffer) byteBuffer).elements()); + CmwHelper.applyCmwData(retrievedData, outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + + TestDataClass.setCmwCompatibilityMode(false); + cmwLightSerialiser.setBuffer(byteBuffer); + } + + public static void checkCmwLightVsCmwIdentityForward(final TestDataClass inputObject, TestDataClass outputObject) { + final DataSerializer cmwSerializer = DataFactory.createDataSerializer(); + TestDataClass.setCmwCompatibilityMode(true); + + outputObject.clear(); + byteBuffer.reset(); + CmwLightHelper.serialiseCustom(cmwLightSerialiser, inputObject); + final int nBytesCmwLight = byteBuffer.position(); + + final Data cmwData = CmwHelper.getCmwData(inputObject); + final byte[] cmwBuffer = cmwSerializer.serializeToBinary(cmwData); + FastByteBuffer wrapCmwBuffer = FastByteBuffer.wrap(cmwBuffer); + LOGGER.atInfo().addArgument(cmwBuffer.length).addArgument(nBytesCmwLight).log("forward compatibility check: CMW nBytes = {} vs. CmwLight serialiser nBytes = {}"); + if (cmwBuffer.length != nBytesCmwLight) { + throw new IllegalStateException("CMW byte buffer length = " + cmwBuffer.length + " vs. CmwLight byte buffer length = " + nBytesCmwLight); + } + + wrapCmwBuffer.reset(); + cmwLightSerialiser.setBuffer(wrapCmwBuffer); + final Data retrievedData = cmwSerializer.deserializeFromBinary(cmwBuffer); + CmwHelper.applyCmwData(retrievedData, outputObject); + + // keep: checks serialised data structure + // wrapCmwBuffer.reset(); + // final WireDataFieldDescription fieldRoot = CmwLightHelper.deserialiseMap(cmwLightSerialiser); + // fieldRoot.printFieldStructure(); + + // N.B. cannot use custom deserialiser since entry order seems to be arbitrary in CMW Data object + wrapCmwBuffer.reset(); + ioSerialiser.deserialiseObject(outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + + TestDataClass.setCmwCompatibilityMode(false); + cmwLightSerialiser.setBuffer(byteBuffer); + } +*/ + public static int checkCustomSerialiserIdentity(final TestDataClass inputObject, TestDataClass outputObject) { + outputObject.clear(); + byteBuffer.reset(); + CmwLightHelper.serialiseCustom(cmwLightSerialiser, inputObject); + final int nBytesCmwLight = byteBuffer.position(); + + // keep: checks serialised data structure + // byteBuffer.reset(); + // final WireDataFieldDescription fieldRoot = CmwLightHelper.deserialiseMap(cmwLightSerialiser); + // fieldRoot.printFieldStructure(); + + byteBuffer.reset(); + CmwLightHelper.deserialiseCustom(cmwLightSerialiser, outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + return nBytesCmwLight; + } + + public static int checkSerialiserIdentity(final TestDataClass inputObject, TestDataClass outputObject) { + outputObject.clear(); + byteBuffer.reset(); + + ioSerialiser.serialiseObject(inputObject); + + // CmwLightHelper.serialiseCustom(cmwLightSerialiser, inputObject); + final int nBytes = byteBuffer.position(); + + // keep: checks serialised data structure + // byteBuffer.reset(); + // final WireDataFieldDescription fieldRoot = CmwLightHelper.deserialiseMap(cmwLightSerialiser); + // fieldRoot.printFieldStructure(); + + byteBuffer.reset(); + final Object returnedObject = ioSerialiser.deserialiseObject(outputObject); + + assertSame(outputObject, returnedObject, "Deserialisation expected to be in-place"); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + return nBytes; + } + + public static void deserialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo) { + deserialiseCustom(ioSerialiser, pojo, true); + } + + @SuppressWarnings("PMD.ExcessiveMethodLength") + public static void deserialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo, boolean header) { + if (header) { + final ProtocolInfo headerField = ioSerialiser.checkHeaderInfo(); + byteBuffer.position(headerField.getDataStartPosition()); + } + // read 'nEntries' chunk + nEntries = byteBuffer.getInt(); + if (nEntries <= 0) { + throw new IllegalStateException("nEntries = " + nEntries + " <= 0!"); + } + getFieldHeader(ioSerialiser); + pojo.bool1 = ioSerialiser.getBoolean(); + getFieldHeader(ioSerialiser); + pojo.bool2 = ioSerialiser.getBoolean(); + + getFieldHeader(ioSerialiser); + pojo.byte1 = ioSerialiser.getByte(); + getFieldHeader(ioSerialiser); + pojo.byte2 = ioSerialiser.getByte(); + + if (!TestDataClass.isCmwCompatibilityMode()) { // disabled since reference CMW impl does not support char + getFieldHeader(ioSerialiser); + pojo.char1 = ioSerialiser.getChar(); + getFieldHeader(ioSerialiser); + pojo.char2 = ioSerialiser.getChar(); + } else { + // CMW compatibility mode + getFieldHeader(ioSerialiser); + pojo.char1 = (char) ioSerialiser.getShort(); + getFieldHeader(ioSerialiser); + pojo.char2 = (char) ioSerialiser.getShort(); + } + + getFieldHeader(ioSerialiser); + pojo.short1 = ioSerialiser.getShort(); + getFieldHeader(ioSerialiser); + pojo.short2 = ioSerialiser.getShort(); + + getFieldHeader(ioSerialiser); + pojo.int1 = ioSerialiser.getInt(); + getFieldHeader(ioSerialiser); + pojo.int2 = ioSerialiser.getInt(); + + getFieldHeader(ioSerialiser); + pojo.long1 = ioSerialiser.getLong(); + getFieldHeader(ioSerialiser); + pojo.long2 = ioSerialiser.getLong(); + + getFieldHeader(ioSerialiser); + pojo.float1 = ioSerialiser.getFloat(); + getFieldHeader(ioSerialiser); + pojo.float2 = ioSerialiser.getFloat(); + + getFieldHeader(ioSerialiser); + pojo.double1 = ioSerialiser.getDouble(); + getFieldHeader(ioSerialiser); + pojo.double2 = ioSerialiser.getDouble(); + + getFieldHeader(ioSerialiser); + pojo.string1 = ioSerialiser.getString(); + if (!TestDataClass.isCmwCompatibilityMode()) { // disabled since reference CMW impl does not support UTF-8 + getFieldHeader(ioSerialiser); + pojo.string2 = ioSerialiser.getString(); + } + + // 1-dim arrays + getFieldHeader(ioSerialiser); + pojo.boolArray = ioSerialiser.getBooleanArray(); + getFieldHeader(ioSerialiser); + pojo.byteArray = ioSerialiser.getByteArray(); + // getFieldHeader(ioSerialiser); + // pojo.charArray = ioSerialiser.getCharArray(ioSerialiser); + getFieldHeader(ioSerialiser); + pojo.shortArray = ioSerialiser.getShortArray(); + getFieldHeader(ioSerialiser); + pojo.intArray = ioSerialiser.getIntArray(); + getFieldHeader(ioSerialiser); + pojo.longArray = ioSerialiser.getLongArray(); + getFieldHeader(ioSerialiser); + pojo.floatArray = ioSerialiser.getFloatArray(); + getFieldHeader(ioSerialiser); + pojo.doubleArray = ioSerialiser.getDoubleArray(); + getFieldHeader(ioSerialiser); + pojo.stringArray = ioSerialiser.getStringArray(); + + // multidim case + getFieldHeader(ioSerialiser); + pojo.nDimensions = ioSerialiser.getIntArray(); + getFieldHeader(ioSerialiser); + pojo.boolNdimArray = ioSerialiser.getBooleanArray(); + getFieldHeader(ioSerialiser); + pojo.byteNdimArray = ioSerialiser.getByteArray(); + getFieldHeader(ioSerialiser); + pojo.shortNdimArray = ioSerialiser.getShortArray(); + getFieldHeader(ioSerialiser); + pojo.intNdimArray = ioSerialiser.getIntArray(); + getFieldHeader(ioSerialiser); + pojo.longNdimArray = ioSerialiser.getLongArray(); + getFieldHeader(ioSerialiser); + pojo.floatNdimArray = ioSerialiser.getFloatArray(); + getFieldHeader(ioSerialiser); + pojo.doubleNdimArray = ioSerialiser.getDoubleArray(); + + final WireDataFieldDescription field = getFieldHeader(ioSerialiser); + if (field == null) { + // reached the end + return; + } + + if (field.getDataType().equals(DataType.START_MARKER)) { + if (pojo.nestedData == null) { + pojo.nestedData = new TestDataClass(); + } + deserialiseCustom(ioSerialiser, pojo.nestedData, false); + } + } + + public static WireDataFieldDescription deserialiseMap(IoSerialiser ioSerialiser) { + return ioSerialiser.parseIoStream(true); + } + + public static IoBuffer getByteBuffer() { + return byteBuffer; + } + + public static CmwLightSerialiser getCmwLightSerialiser() { + return cmwLightSerialiser; + } + + public static void serialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo) { + serialiseCustom(ioSerialiser, pojo, true); + } + + public static void serialiseCustom(final IoSerialiser ioSerialiser, final TestDataClass pojo, final boolean header) { + if (header) { + ioSerialiser.putHeaderInfo(); + } + + ioSerialiser.put("bool1", pojo.bool1); + ioSerialiser.put("bool2", pojo.bool2); + ioSerialiser.put("byte1", pojo.byte1); + ioSerialiser.put("byte2", pojo.byte2); + if (!TestDataClass.isCmwCompatibilityMode()) { // disabled since reference CMW impl does not support char + ioSerialiser.put("char1", pojo.char1); + ioSerialiser.put("char2", pojo.char2); + } else { + // CMW compatibility mode + ioSerialiser.put("char1", (short) pojo.char1); + ioSerialiser.put("char2", (short) pojo.char2); + } + ioSerialiser.put("short1", pojo.short1); + ioSerialiser.put("short2", pojo.short2); + ioSerialiser.put("int1", pojo.int1); + ioSerialiser.put("int2", pojo.int2); + ioSerialiser.put("long1", pojo.long1); + ioSerialiser.put("long2", pojo.long2); + ioSerialiser.put("float1", pojo.float1); + ioSerialiser.put("float2", pojo.float2); + ioSerialiser.put("double1", pojo.double1); + ioSerialiser.put("double2", pojo.double2); + ioSerialiser.put("string1", pojo.string1); + if (!TestDataClass.isCmwCompatibilityMode()) { // disabled since reference CMW impl does not support UTF-8 + ioSerialiser.put("string2", pojo.string2); + } + + // 1D-arrays + ioSerialiser.put("boolArray", pojo.boolArray, pojo.boolArray.length); + ioSerialiser.put("byteArray", pojo.byteArray, pojo.byteArray.length); + //ioSerialiser.put("charArray", pojo.charArray, pojo.charArray.length); // not supported by CMW + ioSerialiser.put("shortArray", pojo.shortArray, pojo.shortArray.length); + ioSerialiser.put("intArray", pojo.intArray, pojo.intArray.length); + ioSerialiser.put("longArray", pojo.longArray, pojo.longArray.length); + ioSerialiser.put("floatArray", pojo.floatArray, pojo.floatArray.length); + ioSerialiser.put("doubleArray", pojo.doubleArray, pojo.doubleArray.length); + ioSerialiser.put("stringArray", pojo.stringArray, pojo.stringArray.length); + + // multi-dim case + ioSerialiser.put("nDimensions", pojo.nDimensions, pojo.nDimensions.length); + ioSerialiser.put("boolNdimArray", pojo.boolNdimArray, pojo.nDimensions); + ioSerialiser.put("byteNdimArray", pojo.byteNdimArray, pojo.nDimensions); + //ioSerialiser.put("charNdimArray", pojo.nDimensions); // not supported by CMW + ioSerialiser.put("shortNdimArray", pojo.shortNdimArray, pojo.nDimensions); + ioSerialiser.put("intNdimArray", pojo.intNdimArray, pojo.nDimensions); + ioSerialiser.put("longNdimArray", pojo.longNdimArray, pojo.nDimensions); + ioSerialiser.put("floatNdimArray", pojo.floatNdimArray, pojo.nDimensions); + ioSerialiser.put("doubleNdimArray", pojo.doubleNdimArray, pojo.nDimensions); + + if (pojo.nestedData != null) { + final String dataStartMarkerName = "nestedData"; + final WireDataFieldDescription nestedDataMarker = new WireDataFieldDescription(ioSerialiser, null, dataStartMarkerName.hashCode(), dataStartMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(nestedDataMarker); + serialiseCustom(ioSerialiser, pojo.nestedData, false); + ioSerialiser.putEndMarker(nestedDataMarker); + } + + if (header) { + final String dataEndMarkerName = "OBJ_ROOT_END"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); + } + } + + public static void testCustomSerialiserPerformance(final int iterations, final TestDataClass inputObject, final TestDataClass outputObject) { + final long startTime = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + outputObject.clear(); + byteBuffer.reset(); + CmwLightHelper.serialiseCustom(cmwLightSerialiser, inputObject); + + byteBuffer.reset(); + CmwLightHelper.deserialiseCustom(cmwLightSerialiser, outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.position() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.position(), true)) // + .addArgument(diffMillis) // + .log("CmwLight Serializer (custom) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void testPerformancePojo(final int iterations, final TestDataClass inputObject, TestDataClass outputObject) { + cmwLightSerialiser.setPutFieldMetaData(true); + final long startTime = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + outputObject.clear(); + if (i == 1) { + // only stream meta-data the first iteration + cmwLightSerialiser.setPutFieldMetaData(false); + } + byteBuffer.reset(); + ioSerialiser.serialiseObject(inputObject); + + byteBuffer.reset(); + + final Object returnedObject = ioSerialiser.deserialiseObject(outputObject); + + if (outputObject != returnedObject) { // NOPMD - we actually want to compare references + throw new IllegalStateException("Deserialisation expected to be in-place"); + } + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.position() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.position(), true)) // + .addArgument(diffMillis) // + .log("CmwLight Serializer (POJO) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static WireDataFieldDescription testSerialiserPerformanceMap(final int iterations, final TestDataClass inputObject) { + final long startTime = System.nanoTime(); + + WireDataFieldDescription ret = null; + for (int i = 0; i < iterations; i++) { + byteBuffer.reset(); + CmwLightHelper.serialiseCustom(cmwLightSerialiser, inputObject); + byteBuffer.reset(); + ret = CmwLightHelper.deserialiseMap(cmwLightSerialiser); + + if (ret.getDataSize() == 0) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return ret; + } + + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.position() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.position(), true)) // + .addArgument(diffMillis) // + .log("CmwLight Serializer (Map only) throughput = {}/s for {} per test run (took {} ms)"); + return ret; + } + + private static WireDataFieldDescription getFieldHeader(IoSerialiser ioSerialiser) { + if (nEntries == 0) { + return null; + } else if (nEntries <= 0) { + throw new IllegalStateException("nEntries = " + nEntries + " <= 0!"); + } + WireDataFieldDescription field = ioSerialiser.getFieldHeader(); + ioSerialiser.getBuffer().position(field.getDataStartPosition()); + nEntries--; + return field; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/FlatBuffersHelper.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/FlatBuffersHelper.java new file mode 100644 index 00000000..a3490e7a --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/FlatBuffersHelper.java @@ -0,0 +1,355 @@ +package io.opencmw.serialiser.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.benchmark.SerialiserQuickBenchmark; + +import com.google.flatbuffers.ArrayReadWriteBuf; +import com.google.flatbuffers.FlexBuffers; +import com.google.flatbuffers.FlexBuffersBuilder; + +@SuppressWarnings("PMD") // complexity is part of the very large use-case surface that is being tested +public class FlatBuffersHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(SerialiserQuickBenchmark.class); // N.B. SerialiserQuickBenchmark reference on purpose + private static final byte[] rawByteBuffer = new byte[100000]; + + public static ByteBuffer serialiseCustom(FlexBuffersBuilder builder, final TestDataClass pojo) { + return serialiseCustom(builder, pojo, true); + } + + public static ByteBuffer serialiseCustom(FlexBuffersBuilder builder, final TestDataClass pojo, final boolean header) { + final int map = builder.startMap(); + builder.putBoolean("bool1", pojo.bool1); + builder.putBoolean("bool2", pojo.bool2); + + builder.putInt("byte1", pojo.byte1); + builder.putInt("byte2", pojo.byte2); + builder.putInt("short1", pojo.short1); + builder.putInt("short2", pojo.short2); + builder.putInt("char1", pojo.char1); + builder.putInt("char2", pojo.char2); + builder.putInt("int1", pojo.int1); + builder.putInt("int2", pojo.int2); + builder.putInt("long1", pojo.long1); + builder.putInt("long2", pojo.long2); + + builder.putFloat("float1", pojo.float1); + builder.putFloat("float2", pojo.float2); + builder.putFloat("double1", pojo.double1); + builder.putFloat("double2", pojo.double2); + builder.putString("string1", pojo.string1); + builder.putString("string2", pojo.string2); + + // 1D-arrays + final boolean typed = false; + final boolean fixed = false; + int svec = builder.startVector(); + for (int i = 0; i < pojo.boolArray.length; i++) { + builder.putBoolean(pojo.boolArray[i]); + } + builder.endVector("boolArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.byteArray.length; i++) { + builder.putInt(pojo.byteArray[i]); + } + builder.endVector("byteArray", svec, typed, fixed); + + // builder.putFieldHeader("charArray", DataType.CHAR_ARRAY); + // builder.put(pojo.charArray); + + svec = builder.startVector(); + for (int i = 0; i < pojo.shortArray.length; i++) { + builder.putInt(pojo.shortArray[i]); + } + builder.endVector("shortArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.intArray.length; i++) { + builder.putInt(pojo.intArray[i]); + } + builder.endVector("intArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.longArray.length; i++) { + builder.putInt(pojo.longArray[i]); + } + builder.endVector("longArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.floatArray.length; i++) { + builder.putFloat(pojo.floatArray[i]); + } + builder.endVector("floatArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.doubleArray.length; i++) { + builder.putFloat(pojo.doubleArray[i]); + } + builder.endVector("doubleArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.stringArray.length; i++) { + builder.putString(pojo.stringArray[i]); + } + builder.endVector("stringArray", svec, typed, fixed); + + // multi-dim case + svec = builder.startVector(); + for (int i = 0; i < pojo.nDimensions.length; i++) { + builder.putInt(pojo.nDimensions[i]); + } + builder.endVector("nDimensions", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.boolNdimArray.length; i++) { + builder.putBoolean(pojo.boolNdimArray[i]); + } + builder.endVector("boolNdimArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.byteNdimArray.length; i++) { + builder.putInt(pojo.byteNdimArray[i]); + } + builder.endVector("byteNdimArray", svec, typed, fixed); + + // builder.putFieldHeader("charArray", DataType.CHAR_ARRAY); + // builder.put(pojo.charArray); + + svec = builder.startVector(); + for (int i = 0; i < pojo.shortNdimArray.length; i++) { + builder.putInt(pojo.shortNdimArray[i]); + } + builder.endVector("shortNdimArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.intNdimArray.length; i++) { + builder.putInt(pojo.intNdimArray[i]); + } + builder.endVector("intNdimArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.longNdimArray.length; i++) { + builder.putInt(pojo.longNdimArray[i]); + } + builder.endVector("longNdimArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.floatNdimArray.length; i++) { + builder.putFloat(pojo.floatNdimArray[i]); + } + builder.endVector("floatNdimArray", svec, typed, fixed); + + svec = builder.startVector(); + for (int i = 0; i < pojo.doubleNdimArray.length; i++) { + builder.putFloat(pojo.doubleNdimArray[i]); + } + builder.endVector("doubleNdimArray", svec, typed, fixed); + builder.endMap(null, map); + + if (pojo.nestedData != null) { + // final int nestedMap = builder.startMap(); + serialiseCustom(builder, pojo.nestedData, false); + // builder.endMap("nestedData", map); + } + + if (header) { + return builder.finish(); + } + return null; + } + + public static void deserialiseCustom(ByteBuffer buffer, final TestDataClass pojo) { + FlexBuffers.Map map = FlexBuffers.getRoot(new ArrayReadWriteBuf(buffer.array(), buffer.limit())).asMap(); + deserialiseCustom(map, pojo, true); + } + + public static void deserialiseCustom(FlexBuffers.Map map, final TestDataClass pojo, boolean header) { + pojo.bool1 = map.get("bool1").asBoolean(); + pojo.bool2 = map.get("bool2").asBoolean(); + pojo.byte1 = (byte) map.get("byte1").asInt(); + pojo.byte2 = (byte) map.get("byte2").asInt(); + pojo.char1 = (char) map.get("char1").asInt(); + pojo.char2 = (char) map.get("char2").asInt(); + pojo.short1 = (short) map.get("short1").asInt(); + pojo.short2 = (short) map.get("short2").asInt(); + pojo.int1 = map.get("int1").asInt(); + pojo.int2 = map.get("int2").asInt(); + pojo.long1 = map.get("long1").asLong(); + pojo.long2 = map.get("long2").asLong(); + pojo.float1 = (float) map.get("float1").asFloat(); + pojo.float2 = (float) map.get("float2").asFloat(); + pojo.double1 = map.get("double1").asFloat(); + pojo.double2 = map.get("double2").asFloat(); + pojo.string1 = map.get("string1").asString(); + pojo.string2 = map.get("string2").asString(); + + // 1-dim arrays + FlexBuffers.Vector vector; + + vector = map.get("boolArray").asVector(); + pojo.boolArray = new boolean[vector.size()]; + for (int i = 0; i < pojo.boolArray.length; i++) { + pojo.boolArray[i] = vector.get(i).asBoolean(); + } + + vector = map.get("byteArray").asVector(); + pojo.byteArray = new byte[vector.size()]; + for (int i = 0; i < pojo.byteArray.length; i++) { + pojo.byteArray[i] = (byte) vector.get(i).asInt(); + } + + vector = map.get("shortArray").asVector(); + pojo.shortArray = new short[vector.size()]; + for (int i = 0; i < pojo.shortArray.length; i++) { + pojo.shortArray[i] = (short) vector.get(i).asInt(); + } + + // vector = map.get("charArray").asVector(); + // pojo.charArray = new int[vector.size()]; + // for (int i = 0; i < pojo.boolArray.length; i++) { + // pojo.charArray[i] = (char)vector.get(i).asInt(); + // } + + vector = map.get("intArray").asVector(); + pojo.intArray = new int[vector.size()]; + for (int i = 0; i < pojo.intArray.length; i++) { + pojo.intArray[i] = vector.get(i).asInt(); + } + + vector = map.get("longArray").asVector(); + pojo.longArray = new long[vector.size()]; + for (int i = 0; i < pojo.longArray.length; i++) { + pojo.longArray[i] = vector.get(i).asLong(); + } + + vector = map.get("floatArray").asVector(); + pojo.floatArray = new float[vector.size()]; + for (int i = 0; i < pojo.floatArray.length; i++) { + pojo.floatArray[i] = (float) vector.get(i).asFloat(); + } + + vector = map.get("doubleArray").asVector(); + pojo.doubleArray = new double[vector.size()]; + for (int i = 0; i < pojo.doubleArray.length; i++) { + pojo.doubleArray[i] = vector.get(i).asFloat(); + } + + vector = map.get("stringArray").asVector(); + pojo.stringArray = new String[vector.size()]; + for (int i = 0; i < pojo.stringArray.length; i++) { + pojo.stringArray[i] = vector.get(i).asString(); + } + + // multidim case + vector = map.get("nDimensions").asVector(); + pojo.nDimensions = new int[vector.size()]; + for (int i = 0; i < pojo.nDimensions.length; i++) { + pojo.nDimensions[i] = vector.get(i).asInt(); + } + + vector = map.get("boolNdimArray").asVector(); + pojo.boolNdimArray = new boolean[vector.size()]; + for (int i = 0; i < pojo.boolNdimArray.length; i++) { + pojo.boolNdimArray[i] = vector.get(i).asBoolean(); + } + + vector = map.get("byteNdimArray").asVector(); + pojo.byteNdimArray = new byte[vector.size()]; + for (int i = 0; i < pojo.byteNdimArray.length; i++) { + pojo.byteNdimArray[i] = (byte) vector.get(i).asInt(); + } + + vector = map.get("shortNdimArray").asVector(); + pojo.shortNdimArray = new short[vector.size()]; + for (int i = 0; i < pojo.shortNdimArray.length; i++) { + pojo.shortNdimArray[i] = (short) vector.get(i).asInt(); + } + + // vector = map.get("charNdimArray").asVector(); + // pojo.charNdimArray = new int[vector.size()]; + // for (int i = 0; i < pojo.charNdimArray.length; i++) { + // pojo.charNdimArray[i] = (char)vector.get(i).asInt(); + // } + + vector = map.get("intNdimArray").asVector(); + pojo.intNdimArray = new int[vector.size()]; + for (int i = 0; i < pojo.intNdimArray.length; i++) { + pojo.intNdimArray[i] = vector.get(i).asInt(); + } + + vector = map.get("longNdimArray").asVector(); + pojo.longNdimArray = new long[vector.size()]; + for (int i = 0; i < pojo.longNdimArray.length; i++) { + pojo.longNdimArray[i] = vector.get(i).asLong(); + } + + vector = map.get("floatNdimArray").asVector(); + pojo.floatNdimArray = new float[vector.size()]; + for (int i = 0; i < pojo.floatNdimArray.length; i++) { + pojo.floatNdimArray[i] = (float) vector.get(i).asFloat(); + } + + vector = map.get("doubleNdimArray").asVector(); + pojo.doubleNdimArray = new double[vector.size()]; + for (int i = 0; i < pojo.doubleNdimArray.length; i++) { + pojo.doubleNdimArray[i] = vector.get(i).asFloat(); + } + + final FlexBuffers.Map nestedMap = map.get("nestedData").asMap(); + + if (nestedMap != null && nestedMap.size() != 0) { + deserialiseCustom(map.get("nestedData").asMap(), pojo.nestedData, false); + } + } + + public static void testCustomSerialiserPerformance(final int iterations, final TestDataClass inputObject, final TestDataClass outputObject) { + final long startTime = System.nanoTime(); + + ByteBuffer retVal = FlatBuffersHelper.serialiseCustom(new FlexBuffersBuilder(new ArrayReadWriteBuf(rawByteBuffer), FlexBuffersBuilder.BUILDER_FLAG_SHARE_KEYS_AND_STRINGS), inputObject); + for (int i = 0; i < iterations; i++) { + // retVal = FlatBuffersHelper.serialiseCustom(new FlexBuffersBuilder(new ArrayReadWriteBuf(rawByteBuffer), FlexBuffersBuilder.BUILDER_FLAG_SHARE_KEYS_AND_STRINGS), inputObject); + retVal = FlatBuffersHelper.serialiseCustom(new FlexBuffersBuilder(new ArrayReadWriteBuf(rawByteBuffer), FlexBuffersBuilder.BUILDER_FLAG_NONE), inputObject); + + FlatBuffersHelper.deserialiseCustom(retVal, outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((retVal.limit() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(retVal.limit(), true)) // + .addArgument(diffMillis) // + .log("FlatBuffers (custom FlexBuffers) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static int checkCustomSerialiserIdentity(final TestDataClass inputObject, final TestDataClass outputObject) { + //final FlexBuffersBuilder floatBuffersBuilder = new FlexBuffersBuilder(new ArrayReadWriteBuf(rawByteBuffer), FlexBuffersBuilder.BUILDER_FLAG_SHARE_KEYS_AND_STRINGS); + final FlexBuffersBuilder floatBuffersBuilder = new FlexBuffersBuilder(new ArrayReadWriteBuf(rawByteBuffer), FlexBuffersBuilder.BUILDER_FLAG_NONE); + final ByteBuffer retVal = FlatBuffersHelper.serialiseCustom(floatBuffersBuilder, inputObject); + final int nBytesFlatBuffers = retVal.limit(); + + FlatBuffersHelper.deserialiseCustom(retVal, outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + // assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + return nBytesFlatBuffers; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/GenericsHelperTests.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/GenericsHelperTests.java new file mode 100644 index 00000000..e1a743f7 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/GenericsHelperTests.java @@ -0,0 +1,76 @@ +package io.opencmw.serialiser.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class GenericsHelperTests { + @Test + public void testBoxedToPrimitiveConversions() { + assertArrayEquals(new boolean[] { true, false, true }, GenericsHelper.toBoolPrimitive(new Boolean[] { true, false, true })); + assertArrayEquals(new byte[] { (byte) 1, (byte) 0, (byte) 2 }, GenericsHelper.toBytePrimitive(new Byte[] { (byte) 1, (byte) 0, (byte) 2 })); + assertArrayEquals(new char[] { (char) 1, (char) 0, (char) 2 }, GenericsHelper.toCharPrimitive(new Character[] { (char) 1, (char) 0, (char) 2 })); + assertArrayEquals(new short[] { (short) 1, (short) 0, (short) 2 }, GenericsHelper.toShortPrimitive(new Short[] { (short) 1, (short) 0, (short) 2 })); + assertArrayEquals(new int[] { 1, 0, 2 }, GenericsHelper.toIntegerPrimitive(new Integer[] { 1, 0, 2 })); + assertArrayEquals(new long[] { (long) 1, (long) 0, (long) 2 }, GenericsHelper.toLongPrimitive(new Long[] { (long) 1, (long) 0, (long) 2 })); + assertArrayEquals(new float[] { (float) 1, (float) 0, (float) 2 }, GenericsHelper.toFloatPrimitive(new Float[] { (float) 1, (float) 0, (float) 2 })); + assertArrayEquals(new double[] { (double) 1, (double) 0, (double) 2 }, GenericsHelper.toDoublePrimitive(new Double[] { (double) 1, (double) 0, (double) 2 })); + assertArrayEquals(new String[] { "1.0", "0.0", "2.0" }, GenericsHelper.toStringPrimitive(new String[] { "1.0", "0.0", "2.0" })); + + assertArrayEquals(new boolean[] { true, false, true }, GenericsHelper.toPrimitive(new Boolean[] { true, false, true })); + assertArrayEquals(new byte[] { (byte) 1, (byte) 0, (byte) 2 }, GenericsHelper.toPrimitive(new Byte[] { (byte) 1, (byte) 0, (byte) 2 })); + assertArrayEquals(new char[] { (char) 1, (char) 0, (char) 2 }, GenericsHelper.toPrimitive(new Character[] { (char) 1, (char) 0, (char) 2 })); + assertArrayEquals(new short[] { (short) 1, (short) 0, (short) 2 }, GenericsHelper.toPrimitive(new Short[] { (short) 1, (short) 0, (short) 2 })); + assertArrayEquals(new int[] { 1, 0, 2 }, GenericsHelper.toPrimitive(new Integer[] { 1, 0, 2 })); + assertArrayEquals(new long[] { (long) 1, (long) 0, (long) 2 }, GenericsHelper.toPrimitive(new Long[] { (long) 1, (long) 0, (long) 2 })); + assertArrayEquals(new float[] { (float) 1, (float) 0, (float) 2 }, GenericsHelper.toPrimitive(new Float[] { (float) 1, (float) 0, (float) 2 })); + assertArrayEquals(new double[] { (double) 1, (double) 0, (double) 2 }, GenericsHelper.toPrimitive(new Double[] { (double) 1, (double) 0, (double) 2 })); + assertArrayEquals(new String[] { "1.0", "0.0", "2.0" }, GenericsHelper.toStringPrimitive(new String[] { "1.0", "0.0", "2.0" })); + } + + @Test + public void testPrimitiveToObjectConversions() { + assertArrayEquals(new Boolean[] { true, false, true }, GenericsHelper.toObject(new boolean[] { true, false, true })); + assertArrayEquals(new Byte[] { (byte) 1, (byte) 0, (byte) 2 }, GenericsHelper.toObject(new byte[] { (byte) 1, (byte) 0, (byte) 2 })); + assertArrayEquals(new Character[] { (char) 1, (char) 0, (char) 2 }, GenericsHelper.toObject(new char[] { (char) 1, (char) 0, (char) 2 })); + assertArrayEquals(new Short[] { (short) 1, (short) 0, (short) 2 }, GenericsHelper.toObject(new short[] { (short) 1, (short) 0, (short) 2 })); + assertArrayEquals(new Integer[] { 1, 0, 2 }, GenericsHelper.toObject(new int[] { 1, 0, 2 })); + assertArrayEquals(new Long[] { (long) 1, (long) 0, (long) 2 }, GenericsHelper.toObject(new long[] { (long) 1, (long) 0, (long) 2 })); + assertArrayEquals(new Float[] { (float) 1, (float) 0, (float) 2 }, GenericsHelper.toObject(new float[] { (float) 1, (float) 0, (float) 2 })); + assertArrayEquals(new Double[] { (double) 1, (double) 0, (double) 2 }, GenericsHelper.toObject(new double[] { (double) 1, (double) 0, (double) 2 })); + } + + @Test + public void testAnyToDoublePrimitive() { + assertArrayEquals(new double[] { 1.0, 0.0, 1.0 }, GenericsHelper.toDoublePrimitive(new boolean[] { true, false, true })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new byte[] { (byte) 1, (byte) 0, (byte) 2 })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new char[] { (char) 1, (char) 0, (char) 2 })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new short[] { (short) 1, (short) 0, (short) 2 })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new int[] { 1, 0, 2 })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new long[] { (long) 1, (long) 0, (long) 2 })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new float[] { (float) 1, (float) 0, (float) 2 })); + assertArrayEquals(new double[] { 1.0, 0.0, 2.0 }, GenericsHelper.toDoublePrimitive(new String[] { "1.0", "0.0", "2.0" })); + } + + @Test + public void testAnyToStringPrimitive() { + assertArrayEquals(new String[] { "true", "false", "true" }, GenericsHelper.toStringPrimitive(new Boolean[] { true, false, true })); + assertArrayEquals(new String[] { "1", "0", "2" }, GenericsHelper.toStringPrimitive(new Byte[] { (byte) 1, (byte) 0, (byte) 2 })); + assertArrayEquals(new String[] { "A", "B", "C" }, GenericsHelper.toStringPrimitive(new Character[] { (char) 65, (char) 66, (char) 67 })); + assertArrayEquals(new String[] { "1", "0", "2" }, GenericsHelper.toStringPrimitive(new Short[] { (short) 1, (short) 0, (short) 2 })); + assertArrayEquals(new String[] { "1", "0", "2" }, GenericsHelper.toStringPrimitive(new Integer[] { 1, 0, 2 })); + assertArrayEquals(new String[] { "1", "0", "2" }, GenericsHelper.toStringPrimitive(new Long[] { (long) 1, (long) 0, (long) 2 })); + assertArrayEquals(new String[] { "1.0", "0.0", "2.0" }, GenericsHelper.toStringPrimitive(new Float[] { (float) 1, (float) 0, (float) 2 })); + assertArrayEquals(new String[] { "1.0", "0.0", "2.0" }, GenericsHelper.toStringPrimitive(new Double[] { (double) 1, (double) 0, (double) 2 })); + assertArrayEquals(new String[] { "1.0", "0.0", "2.0" }, GenericsHelper.toStringPrimitive(new String[] { "1.0", "0.0", "2.0" })); + assertArrayEquals(new String[] {}, GenericsHelper.toStringPrimitive(new String[] {})); + } + + @Test + public void testHelper() { + assertThrows(IllegalArgumentException.class, () -> GenericsHelper.toBytePrimitive(null)); + assertDoesNotThrow(() -> GenericsHelper.toBytePrimitive(new Integer[] {})); + assertThrows(IllegalArgumentException.class, () -> GenericsHelper.toBytePrimitive(new Integer[] { null })); + assertThrows(IllegalArgumentException.class, () -> GenericsHelper.toBytePrimitive(new Integer[] { 1 })); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/JsonHelper.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/JsonHelper.java new file mode 100644 index 00000000..ffc34800 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/JsonHelper.java @@ -0,0 +1,278 @@ +package io.opencmw.serialiser.utils; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.benchmark.SerialiserQuickBenchmark; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.JsonSerialiser; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +import com.jsoniter.JsonIterator; +import com.jsoniter.extra.PreciseFloatSupport; +import com.jsoniter.output.EncodingMode; +import com.jsoniter.output.JsonStream; +import com.jsoniter.spi.DecodingMode; +import com.jsoniter.spi.JsonException; + +public final class JsonHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(SerialiserQuickBenchmark.class); // N.B. SerialiserQuickBenchmark reference on purpose + private static final IoBuffer byteBuffer = new FastByteBuffer(1000000); + // private static final IoBuffer byteBuffer = new ByteBuffer(20000); + private static final JsonSerialiser jsonSerialiser = new JsonSerialiser(byteBuffer); + + public static int checkCustomSerialiserIdentity(final TestDataClass inputObject, TestDataClass outputObject) { + outputObject.clear(); + byteBuffer.reset(); + JsonHelper.serialiseCustom(jsonSerialiser, inputObject); + byteBuffer.flip(); + + // keep: checks serialised data structure + // byteBuffer.reset(); + // final WireDataFieldDescription fieldRoot = CmwLightHelper.deserialiseMap(cmwLightSerialiser); + // fieldRoot.printFieldStructure(); + + jsonSerialiser.deserialiseObject(outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + return byteBuffer.limit(); + } + + public static int checkSerialiserIdentity(final TestDataClass inputObject, TestDataClass outputObject) { + outputObject.clear(); + // JsonIterator.setMode(DecodingMode.DYNAMIC_MODE_AND_MATCH_FIELD_WITH_HASH); + // JsonStream.setMode(EncodingMode.DYNAMIC_MODE); + // JsonIterator.setMode(DecodingMode.REFLECTION_MODE); + // JsonStream.setIndentionStep(2); // sets line-breaks and indentation (more human readable) + //Base64Support.enable(); + //Base64FloatSupport.enableEncodersAndDecoders(); + JsonStream.setMode(EncodingMode.DYNAMIC_MODE); + + try { + PreciseFloatSupport.enable(); + } catch (JsonException e) { + // swallow subsequent enabling exceptions (function is guarded and supposed to be called only once) + } + + byteBuffer.reset(); + jsonSerialiser.serialiseObject(inputObject); + + byteBuffer.flip(); + + jsonSerialiser.deserialiseObject(outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + return byteBuffer.limit(); + } + + public static IoBuffer getByteBuffer() { + return byteBuffer; + } + + public static void serialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo) { + serialiseCustom(ioSerialiser, pojo, true); + } + + public static void serialiseCustom(final IoSerialiser ioSerialiser, final TestDataClass pojo, final boolean header) { + if (header) { + ioSerialiser.putHeaderInfo(); + } + + ioSerialiser.put("bool1", pojo.bool1); + ioSerialiser.put("bool2", pojo.bool2); + ioSerialiser.put("byte1", pojo.byte1); + ioSerialiser.put("byte2", pojo.byte2); + ioSerialiser.put("char1", pojo.char1); + ioSerialiser.put("char2", pojo.char2); + ioSerialiser.put("short1", pojo.short1); + ioSerialiser.put("short2", pojo.short2); + ioSerialiser.put("int1", pojo.int1); + ioSerialiser.put("int2", pojo.int2); + ioSerialiser.put("long1", pojo.long1); + ioSerialiser.put("long2", pojo.long2); + ioSerialiser.put("float1", pojo.float1); + ioSerialiser.put("float2", pojo.float2); + ioSerialiser.put("double1", pojo.double1); + ioSerialiser.put("double2", pojo.double2); + ioSerialiser.put("string1", pojo.string1); + ioSerialiser.put("string2", pojo.string2); + + // 1D-arrays + ioSerialiser.put("boolArray", pojo.boolArray, pojo.boolArray.length); + ioSerialiser.put("byteArray", pojo.byteArray, pojo.byteArray.length); + //ioSerialiser.put("charArray", pojo.charArray, pojo.charArray.lenght); + ioSerialiser.put("shortArray", pojo.shortArray, pojo.shortArray.length); + ioSerialiser.put("intArray", pojo.intArray, pojo.intArray.length); + ioSerialiser.put("longArray", pojo.longArray, pojo.longArray.length); + ioSerialiser.put("floatArray", pojo.floatArray, pojo.floatArray.length); + ioSerialiser.put("doubleArray", pojo.doubleArray, pojo.doubleArray.length); + ioSerialiser.put("stringArray", pojo.stringArray, pojo.stringArray.length); + + // multi-dim case + ioSerialiser.put("nDimensions", pojo.nDimensions, pojo.nDimensions.length); + ioSerialiser.put("boolNdimArray", pojo.boolNdimArray, pojo.nDimensions); + ioSerialiser.put("byteNdimArray", pojo.byteNdimArray, pojo.nDimensions); + //ioSerialiser.put("charNdimArray", pojo.nDimensions); + ioSerialiser.put("shortNdimArray", pojo.shortNdimArray, pojo.nDimensions); + ioSerialiser.put("intNdimArray", pojo.intNdimArray, pojo.nDimensions); + ioSerialiser.put("longNdimArray", pojo.longNdimArray, pojo.nDimensions); + ioSerialiser.put("floatNdimArray", pojo.floatNdimArray, pojo.nDimensions); + ioSerialiser.put("doubleNdimArray", pojo.doubleNdimArray, pojo.nDimensions); + + if (pojo.nestedData != null) { + final String dataStartMarkerName = "nestedData"; + final WireDataFieldDescription nestedDataMarker = new WireDataFieldDescription(ioSerialiser, null, dataStartMarkerName.hashCode(), dataStartMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(nestedDataMarker); + serialiseCustom(ioSerialiser, pojo.nestedData, false); + ioSerialiser.putEndMarker(nestedDataMarker); + } + + if (header) { + final String dataEndMarkerName = "OBJ_ROOT_END"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); + } + } + + public static void testCustomSerialiserPerformance(final int iterations, final TestDataClass inputObject, final TestDataClass outputObject) { + final long startTime = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + outputObject.clear(); + byteBuffer.reset(); + JsonHelper.serialiseCustom(jsonSerialiser, inputObject); + + byteBuffer.flip(); + jsonSerialiser.deserialiseObject(outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.limit() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.limit(), true)) // + .addArgument(diffMillis) // + .log("JSON Serializer (custom) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void testPerformancePojo(final int iterations, final TestDataClass inputObject, TestDataClass outputObject) { + // works with all classes, in particular those having private fields + JsonStream.setMode(EncodingMode.REFLECTION_MODE); + JsonIterator.setMode(DecodingMode.REFLECTION_MODE); + + outputObject.clear(); + final long startTime = System.nanoTime(); + testPerformancePojoNoPrintout(iterations, inputObject, outputObject); + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.limit() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.limit(), true)) // + .addArgument(diffMillis) // + .log("JSON Serializer (POJO, reflection-only) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void testPerformancePojoCodeGen(final int iterations, final TestDataClass inputObject, TestDataClass outputObject) { + // N.B. works only for all-public fields + JsonStream.setMode(EncodingMode.DYNAMIC_MODE); + JsonIterator.setMode(DecodingMode.DYNAMIC_MODE_AND_MATCH_FIELD_WITH_HASH); + + outputObject.clear(); + final long startTime = System.nanoTime(); + testPerformancePojoNoPrintout(iterations, inputObject, outputObject); + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.limit() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.limit(), true)) // + .addArgument(diffMillis) // + .log("JSON Serializer (POJO, code-gen) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void testPerformancePojoNoPrintout(final int iterations, final TestDataClass inputObject, TestDataClass outputObject) { + for (int i = 0; i < iterations; i++) { + byteBuffer.reset(); + outputObject.clear(); + jsonSerialiser.serialiseObject(inputObject); + + byteBuffer.flip(); + jsonSerialiser.deserialiseObject(outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + } + + public static void testSerialiserPerformanceMap(final int iterations, final TestDataClass inputObject) { + byteBuffer.reset(); + jsonSerialiser.serialiseObject(inputObject); + + byteBuffer.flip(); + + final long startTime = System.nanoTime(); + + final WireDataFieldDescription wireDataHeader = jsonSerialiser.parseIoStream(true); + // wireDataHeader.printFieldStructure(); + + if (wireDataHeader == null || wireDataHeader.getChildren().get(0) == null || wireDataHeader.getChildren().get(0).findChildField("string1") == null + || !((WireDataFieldDescription) wireDataHeader.getChildren().get(0).findChildField("string1")).data().equals(inputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + + if (iterations <= 1) { + // JMH use-case + return; + } + + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.limit() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.limit(), true)) // + .addArgument(diffMillis) // + .log("JSON Serializer (Map only) throughput = {}/s for {} per test run (took {} ms)"); + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/SerialiserHelper.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/SerialiserHelper.java new file mode 100644 index 00000000..f9fa04f3 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/SerialiserHelper.java @@ -0,0 +1,355 @@ +package io.opencmw.serialiser.utils; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.serialiser.DataType; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.IoSerialiser; +import io.opencmw.serialiser.benchmark.SerialiserQuickBenchmark; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.WireDataFieldDescription; + +@SuppressWarnings("PMD") // complexity is part of the very large use-case surface that is being tested +public final class SerialiserHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(SerialiserQuickBenchmark.class); // N.B. SerialiserQuickBenchmark reference on purpose + private static final IoBuffer byteBuffer = new FastByteBuffer(100000); + + // private static final IoBuffer byteBuffer = new ByteBuffer(20000); + private static final BinarySerialiser binarySerialiser = new BinarySerialiser(byteBuffer); + private static final IoClassSerialiser ioSerialiser = new IoClassSerialiser(byteBuffer, BinarySerialiser.class); + + public static int checkCustomSerialiserIdentity(final TestDataClass inputObject, TestDataClass outputObject) { + outputObject.clear(); + byteBuffer.reset(); + SerialiserHelper.serialiseCustom(binarySerialiser, inputObject); + + byteBuffer.flip(); + + // keep: checks serialised data structure + // byteBuffer.reset(); + // final WireDataFieldDescription fieldRoot = SerialiserHelper.deserialiseMap(byteBuffer); + // fieldRoot.printFieldStructure(); + + SerialiserHelper.deserialiseCustom(binarySerialiser, outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + return byteBuffer.limit(); + } + + public static int checkSerialiserIdentity(final TestDataClass inputObject, TestDataClass outputObject) { + outputObject.clear(); + byteBuffer.reset(); + + ioSerialiser.serialiseObject(inputObject); + + // SerialiserHelper.serialiseCustom(byteBuffer, inputObject); + + byteBuffer.flip(); + // keep: checks serialised data structure + // byteBuffer.reset(); + // final WireDataFieldDescription fieldRoot = SerialiserHelper.deserialiseMap(byteBuffer); + // fieldRoot.printFieldStructure(); + + outputObject = ioSerialiser.deserialiseObject(outputObject); + + // second test - both vectors should have the same initial values after serialise/deserialise + assertArrayEquals(inputObject.stringArray, outputObject.stringArray); + + assertEquals(inputObject, outputObject, "TestDataClass input-output equality"); + + return byteBuffer.limit(); + } + + public static void deserialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo) { + deserialiseCustom(ioSerialiser, pojo, true); + } + + public static void deserialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo, boolean header) { + if (header) { + ioSerialiser.checkHeaderInfo(); + } + + getFieldHeader(ioSerialiser); + pojo.bool1 = ioSerialiser.getBoolean(); + getFieldHeader(ioSerialiser); + pojo.bool2 = ioSerialiser.getBoolean(); + + getFieldHeader(ioSerialiser); + pojo.byte1 = ioSerialiser.getByte(); + getFieldHeader(ioSerialiser); + pojo.byte2 = ioSerialiser.getByte(); + + getFieldHeader(ioSerialiser); + pojo.char1 = ioSerialiser.getChar(); + getFieldHeader(ioSerialiser); + pojo.char2 = ioSerialiser.getChar(); + + getFieldHeader(ioSerialiser); + pojo.short1 = ioSerialiser.getShort(); + getFieldHeader(ioSerialiser); + pojo.short2 = ioSerialiser.getShort(); + + getFieldHeader(ioSerialiser); + pojo.int1 = ioSerialiser.getInt(); + getFieldHeader(ioSerialiser); + pojo.int2 = ioSerialiser.getInt(); + + getFieldHeader(ioSerialiser); + pojo.long1 = ioSerialiser.getLong(); + getFieldHeader(ioSerialiser); + pojo.long2 = ioSerialiser.getLong(); + + getFieldHeader(ioSerialiser); + pojo.float1 = ioSerialiser.getFloat(); + getFieldHeader(ioSerialiser); + pojo.float2 = ioSerialiser.getFloat(); + + getFieldHeader(ioSerialiser); + pojo.double1 = ioSerialiser.getDouble(); + getFieldHeader(ioSerialiser); + pojo.double2 = ioSerialiser.getDouble(); + + getFieldHeader(ioSerialiser); + pojo.string1 = ioSerialiser.getString(); + getFieldHeader(ioSerialiser); + pojo.string2 = ioSerialiser.getString(); + + // 1-dim arrays + getFieldHeader(ioSerialiser); + pojo.boolArray = ioSerialiser.getBooleanArray(); + getFieldHeader(ioSerialiser); + pojo.byteArray = ioSerialiser.getByteArray(); + //getFieldHeader(ioSerialiser); + //pojo.charArray = ioSerialiser.getCharArray(ioSerialiser); + getFieldHeader(ioSerialiser); + pojo.shortArray = ioSerialiser.getShortArray(); + getFieldHeader(ioSerialiser); + pojo.intArray = ioSerialiser.getIntArray(); + getFieldHeader(ioSerialiser); + pojo.longArray = ioSerialiser.getLongArray(); + getFieldHeader(ioSerialiser); + pojo.floatArray = ioSerialiser.getFloatArray(); + getFieldHeader(ioSerialiser); + pojo.doubleArray = ioSerialiser.getDoubleArray(); + getFieldHeader(ioSerialiser); + pojo.stringArray = ioSerialiser.getStringArray(); + + // multidim case + getFieldHeader(ioSerialiser); + pojo.nDimensions = ioSerialiser.getIntArray(); + getFieldHeader(ioSerialiser); + pojo.boolNdimArray = ioSerialiser.getBooleanArray(); + getFieldHeader(ioSerialiser); + pojo.byteNdimArray = ioSerialiser.getByteArray(); + getFieldHeader(ioSerialiser); + pojo.shortNdimArray = ioSerialiser.getShortArray(); + getFieldHeader(ioSerialiser); + pojo.intNdimArray = ioSerialiser.getIntArray(); + getFieldHeader(ioSerialiser); + pojo.longNdimArray = ioSerialiser.getLongArray(); + getFieldHeader(ioSerialiser); + pojo.floatNdimArray = ioSerialiser.getFloatArray(); + getFieldHeader(ioSerialiser); + pojo.doubleNdimArray = ioSerialiser.getDoubleArray(); + + final WireDataFieldDescription field = getFieldHeader(ioSerialiser); + if (field.getDataType().equals(DataType.START_MARKER)) { + if (pojo.nestedData == null) { + pojo.nestedData = new TestDataClass(); + } + deserialiseCustom(ioSerialiser, pojo.nestedData, false); + + } else if (!field.getDataType().equals(DataType.END_MARKER)) { + throw new IllegalStateException("format error/unexpected tag with data type = " + field.getDataType() + " and field name = " + field.getFieldName()); + } + } + + public static WireDataFieldDescription deserialiseMap(IoSerialiser ioSerialiser) { + return ioSerialiser.parseIoStream(true); + } + + public static BinarySerialiser getBinarySerialiser() { + return binarySerialiser; + } + + public static IoBuffer getByteBuffer() { + return byteBuffer; + } + + public static void serialiseCustom(IoSerialiser ioSerialiser, final TestDataClass pojo) { + serialiseCustom(ioSerialiser, pojo, true); + } + + public static void serialiseCustom(final IoSerialiser ioSerialiser, final TestDataClass pojo, final boolean header) { + if (header) { + ioSerialiser.putHeaderInfo(); + } + + ioSerialiser.put("bool1", pojo.bool1); + ioSerialiser.put("bool2", pojo.bool2); + ioSerialiser.put("byte1", pojo.byte1); + ioSerialiser.put("byte2", pojo.byte2); + ioSerialiser.put("char1", pojo.char1); + ioSerialiser.put("char2", pojo.char2); + ioSerialiser.put("short1", pojo.short1); + ioSerialiser.put("short2", pojo.short2); + ioSerialiser.put("int1", pojo.int1); + ioSerialiser.put("int2", pojo.int2); + ioSerialiser.put("long1", pojo.long1); + ioSerialiser.put("long2", pojo.long2); + ioSerialiser.put("float1", pojo.float1); + ioSerialiser.put("float2", pojo.float2); + ioSerialiser.put("double1", pojo.double1); + ioSerialiser.put("double2", pojo.double2); + ioSerialiser.put("string1", pojo.string1); + ioSerialiser.put("string2", pojo.string2); + + // 1D-arrays + ioSerialiser.put("boolArray", pojo.boolArray, pojo.boolArray.length); + ioSerialiser.put("byteArray", pojo.byteArray, pojo.byteArray.length); + //ioSerialiser.put("charArray", pojo.charArray, pojo.charArray.lenght); + ioSerialiser.put("shortArray", pojo.shortArray, pojo.shortArray.length); + ioSerialiser.put("intArray", pojo.intArray, pojo.intArray.length); + ioSerialiser.put("longArray", pojo.longArray, pojo.longArray.length); + ioSerialiser.put("floatArray", pojo.floatArray, pojo.floatArray.length); + ioSerialiser.put("doubleArray", pojo.doubleArray, pojo.doubleArray.length); + ioSerialiser.put("stringArray", pojo.stringArray, pojo.stringArray.length); + + // multi-dim case + ioSerialiser.put("nDimensions", pojo.nDimensions, pojo.nDimensions.length); + ioSerialiser.put("boolNdimArray", pojo.boolNdimArray, pojo.nDimensions); + ioSerialiser.put("byteNdimArray", pojo.byteNdimArray, pojo.nDimensions); + //ioSerialiser.put("charNdimArray", pojo.nDimensions); + ioSerialiser.put("shortNdimArray", pojo.shortNdimArray, pojo.nDimensions); + ioSerialiser.put("intNdimArray", pojo.intNdimArray, pojo.nDimensions); + ioSerialiser.put("longNdimArray", pojo.longNdimArray, pojo.nDimensions); + ioSerialiser.put("floatNdimArray", pojo.floatNdimArray, pojo.nDimensions); + ioSerialiser.put("doubleNdimArray", pojo.doubleNdimArray, pojo.nDimensions); + + if (pojo.nestedData != null) { + final String dataStartMarkerName = "nestedData"; + final WireDataFieldDescription nestedDataMarker = new WireDataFieldDescription(ioSerialiser, null, dataStartMarkerName.hashCode(), dataStartMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putStartMarker(nestedDataMarker); + serialiseCustom(ioSerialiser, pojo.nestedData, false); + ioSerialiser.putEndMarker(nestedDataMarker); + } + + if (header) { + final String dataEndMarkerName = "OBJ_ROOT_END"; + final WireDataFieldDescription dataEndMarker = new WireDataFieldDescription(ioSerialiser, null, dataEndMarkerName.hashCode(), dataEndMarkerName, DataType.START_MARKER, -1, -1, -1); + ioSerialiser.putEndMarker(dataEndMarker); + } + } + + public static void testCustomSerialiserPerformance(final int iterations, final TestDataClass inputObject, final TestDataClass outputObject) { + final long startTime = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + byteBuffer.reset(); + SerialiserHelper.serialiseCustom(binarySerialiser, inputObject); + + byteBuffer.reset(); + SerialiserHelper.deserialiseCustom(binarySerialiser, outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.position() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.position(), true)) // + .addArgument(diffMillis) // + .log("IO Serializer (custom) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static void testPerformancePojo(final int iterations, final TestDataClass inputObject, TestDataClass outputObject) { + binarySerialiser.setPutFieldMetaData(true); + final long startTime = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + if (i == 1) { + // only stream meta-data the first iteration + binarySerialiser.setPutFieldMetaData(false); + } + byteBuffer.reset(); + ioSerialiser.serialiseObject(inputObject); + + byteBuffer.reset(); + + outputObject = ioSerialiser.deserialiseObject(outputObject); + + if (!inputObject.string1.contentEquals(outputObject.string1)) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return; + } + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.position() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.position(), true)) // + .addArgument(diffMillis) // + .log("IO Serializer (POJO) throughput = {}/s for {} per test run (took {} ms)"); + } + + public static WireDataFieldDescription testSerialiserPerformanceMap(final int iterations, final TestDataClass inputObject) { + final long startTime = System.nanoTime(); + + WireDataFieldDescription ret = null; + for (int i = 0; i < iterations; i++) { + byteBuffer.reset(); + SerialiserHelper.serialiseCustom(binarySerialiser, inputObject); + byteBuffer.reset(); + ret = SerialiserHelper.deserialiseMap(binarySerialiser); + + if (ret.getDataSize() == 0) { + // quick check necessary so that the above is not optimised by the Java JIT compiler to NOP + throw new IllegalStateException("data mismatch"); + } + } + if (iterations <= 1) { + // JMH use-case + return ret; + } + + final long stopTime = System.nanoTime(); + + final double diffMillis = TimeUnit.NANOSECONDS.toMillis(stopTime - startTime); + final double byteCount = iterations * ((byteBuffer.position() / diffMillis) * 1e3); + LOGGER.atInfo().addArgument(SerialiserQuickBenchmark.humanReadableByteCount((long) byteCount, true)) // + .addArgument(SerialiserQuickBenchmark.humanReadableByteCount(byteBuffer.position(), true)) // + .addArgument(diffMillis) // + .log("IO Serializer (Map only) throughput = {}/s for {} per test run (took {} ms)"); + return ret; + } + + private static WireDataFieldDescription getFieldHeader(IoSerialiser ioSerialiser) { + WireDataFieldDescription field = ioSerialiser.getFieldHeader(); + ioSerialiser.getBuffer().position(field.getDataStartPosition()); + return field; + } +} diff --git a/serialiser/src/test/java/io/opencmw/serialiser/utils/TestDataClass.java b/serialiser/src/test/java/io/opencmw/serialiser/utils/TestDataClass.java new file mode 100644 index 00000000..dfa2f013 --- /dev/null +++ b/serialiser/src/test/java/io/opencmw/serialiser/utils/TestDataClass.java @@ -0,0 +1,488 @@ +package io.opencmw.serialiser.utils; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.util.Arrays; +import java.util.Objects; + +import org.opentest4j.AssertionFailedError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("PMD") // complexity is part of the very large use-case surface that is being tested +public class TestDataClass { + private static final Logger LOGGER = LoggerFactory.getLogger(TestDataClass.class); + private static transient boolean cmwCompatibilityMode = false; + + public boolean bool1; + public boolean bool2; + public byte byte1; + public byte byte2; + public char char1; + public char char2; + public short short1; + public short short2; + public int int1; + public int int2; + public long long1; + public long long2; + public float float1; + public float float2; + public double double1; + public double double2; + public String string1; + public String string2; + + // 1-dim arrays + public boolean[] boolArray; + public byte[] byteArray; + // public char[] charArray; + public short[] shortArray; + public int[] intArray; + public long[] longArray; + public float[] floatArray; + public double[] doubleArray; + public String[] stringArray; + + // generic n-dim arrays - N.B. striding-arrays: low-level format is the same except of 'nDimension' descriptor + public int[] nDimensions; + public boolean[] boolNdimArray; + public byte[] byteNdimArray; + // public char[] charNdimArray; + public short[] shortNdimArray; + public int[] intNdimArray; + public long[] longNdimArray; + public float[] floatNdimArray; + public double[] doubleNdimArray; + + public TestDataClass nestedData; + + public TestDataClass() { + this(-1, -1, -1); + } + + /** + * @param nSizePrimitives size of primitive arrays (smaller 0: do not initialise fields/allocate arrays) + * @param nSizeString size of String[] array (smaller 0: do not initialise fields/allocate arrays) + * @param nestedClassRecursion how many nested sub-classes should be allocated + */ + public TestDataClass(final int nSizePrimitives, final int nSizeString, final int nestedClassRecursion) { + if (nestedClassRecursion > 0) { + nestedData = new TestDataClass(nSizePrimitives, nSizeString, nestedClassRecursion - 1); + nestedData.init(nSizePrimitives + 1, nSizeString + 1); //N.B. '+1' to have different sizes for nested classes + } + + init(nSizePrimitives, nSizeString); + } + + public final void clear() { + bool1 = false; + bool2 = false; + byte1 = 0; + byte2 = 0; + char1 = 0; + char2 = 0; + short1 = 0; + short2 = 0; + int1 = 0; + int2 = 0; + long1 = 0; + long2 = 0; + float1 = 0; + float2 = 0; + double1 = 0; + double2 = 0; + + string1 = null; + string2 = null; + + // reset 1-dim arrays + boolArray = null; + byteArray = null; + // charArray = null; + shortArray = null; + intArray = null; + longArray = null; + floatArray = null; + doubleArray = null; + stringArray = null; + + // reset n-dim arrays + nDimensions = null; + boolNdimArray = null; + byteNdimArray = null; + // charNdimArray = null; + shortNdimArray = null; + intNdimArray = null; + longNdimArray = null; + floatNdimArray = null; + doubleNdimArray = null; + + nestedData = null; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TestDataClass)) { + LOGGER.atError().addArgument(obj).log("incompatible object type of obj = '{}'"); + return false; + } + final TestDataClass other = (TestDataClass) obj; + boolean returnState = true; + if (this.bool1 != other.bool1) { + LOGGER.atError().addArgument("bool1").addArgument(this.bool1).addArgument(other.bool1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.bool2 != other.bool2) { + LOGGER.atError().addArgument("bool2").addArgument(this.bool2).addArgument(other.bool2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.byte1 != other.byte1) { + LOGGER.atError().addArgument("byte1").addArgument(this.byte1).addArgument(other.byte1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.byte2 != other.byte2) { + LOGGER.atError().addArgument("byte2").addArgument(this.byte2).addArgument(other.byte2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.char1 != other.char1) { + LOGGER.atError().addArgument("char1").addArgument(this.char1).addArgument(other.char1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.char2 != other.char2) { + LOGGER.atError().addArgument("char2").addArgument(this.char2).addArgument(other.char2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.short1 != other.short1) { + LOGGER.atError().addArgument("short1").addArgument(this.short1).addArgument(other.short1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.short2 != other.short2) { + LOGGER.atError().addArgument("short2").addArgument(this.short2).addArgument(other.short2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.int1 != other.int1) { + LOGGER.atError().addArgument("int1").addArgument(this.int1).addArgument(other.int1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.int2 != other.int2) { + LOGGER.atError().addArgument("int2").addArgument(this.int2).addArgument(other.int2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.float1 != other.float1) { + LOGGER.atError().addArgument("float1").addArgument(this.float1).addArgument(other.float1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.float2 != other.float2) { + LOGGER.atError().addArgument("float2").addArgument(this.float2).addArgument(other.float2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.double1 != other.double1) { + LOGGER.atError().addArgument("double1").addArgument(this.double1).addArgument(other.double1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (this.double2 != other.double2) { + LOGGER.atError().addArgument("double2").addArgument(this.double2).addArgument(other.double2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (!Objects.equals(string1, other.string1)) { + LOGGER.atError().addArgument("string1").addArgument(this.string1).addArgument(other.string1) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + if (!isCmwCompatibilityMode()) { + if (!Objects.equals(string2, other.string2)) { + LOGGER.atError().addArgument("string2").addArgument(this.string2).addArgument(other.string2) // + .log("field '{}' does not match '{}' vs '{}'"); + returnState = false; + } + } + + // test 1D-arrays + try { + assertArrayEquals(this.boolArray, other.boolArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("boolArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.byteArray, other.byteArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("byteArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + //try { + // assertArrayEquals(this.charArray, other.charArray); + //} catch(AssertionFailedError e) { + // LOGGER.atError().addArgument("charArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + // returnState = false; + //} + try { + assertArrayEquals(this.shortArray, other.shortArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("shortArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.intArray, other.intArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("intArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.longArray, other.longArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("longArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.floatArray, other.floatArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("floatArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.doubleArray, other.doubleArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("doubleArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.stringArray, other.stringArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("doubleArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + + // test n-dimensional -arrays + try { + assertArrayEquals(this.nDimensions, other.nDimensions); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("nDimensions").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.boolNdimArray, other.boolNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("boolNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.byteNdimArray, other.byteNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("byteNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + //try { + // assertArrayEquals(this.charNdimArray, other.charNdimArray); + //} catch(AssertionFailedError e) { + // LOGGER.atError().addArgument("charNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + // returnState = false; + //} + try { + assertArrayEquals(this.shortNdimArray, other.shortNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("shortNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.intNdimArray, other.intNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("intNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.longNdimArray, other.longNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("longNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.floatNdimArray, other.floatNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("floatNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + try { + assertArrayEquals(this.doubleNdimArray, other.doubleNdimArray); + } catch (AssertionFailedError e) { + LOGGER.atError().addArgument("doubleNdimArray").addArgument(e.getMessage()).log("field '{}' does not match '{}'"); + returnState = false; + } + + // check for nested data content + if (this.nestedData != null) { + returnState = returnState & this.nestedData.equals(other.nestedData); + } else if (other.nestedData != null) { + LOGGER.atError().addArgument("nestedData").addArgument(this.nestedData).addArgument(other.nestedData).log("field '{}' error: this.nestedData == null ({}) && other.nestedData != null ({})"); + returnState = false; + } + + return returnState; + } + + @Override + public int hashCode() { + int result = Objects.hash(bool1, bool2, byte1, byte2, char1, char2, short1, short2, int1, int2, long1, long2, float1, float2, double1, double2, string1, string2, nestedData); + result = 31 * result + Arrays.hashCode(boolArray); + result = 31 * result + Arrays.hashCode(byteArray); + result = 31 * result + Arrays.hashCode(shortArray); + result = 31 * result + Arrays.hashCode(intArray); + result = 31 * result + Arrays.hashCode(longArray); + result = 31 * result + Arrays.hashCode(floatArray); + result = 31 * result + Arrays.hashCode(doubleArray); + result = 31 * result + Arrays.hashCode(stringArray); + result = 31 * result + Arrays.hashCode(nDimensions); + result = 31 * result + Arrays.hashCode(boolNdimArray); + result = 31 * result + Arrays.hashCode(byteNdimArray); + result = 31 * result + Arrays.hashCode(shortNdimArray); + result = 31 * result + Arrays.hashCode(intNdimArray); + result = 31 * result + Arrays.hashCode(longNdimArray); + result = 31 * result + Arrays.hashCode(floatNdimArray); + result = 31 * result + Arrays.hashCode(doubleNdimArray); + return result; + } + + public final void init(final int nSizePrimitives, final int nSizeString) { + if (nSizePrimitives >= 0) { + bool1 = true; + bool2 = false; + byte1 = 10; + byte2 = -100; + char1 = 'a'; + char2 = 'Z'; + short1 = 20; + short2 = -200; + int1 = 30; + int2 = -300; + long1 = 40; + long2 = -400; + float1 = 50.5f; + float2 = -500.5f; + double1 = 60.6; + double2 = -600.6; + + string1 = "Hello World!"; + string2 = "Γειά σου Κόσμε!"; + + // allocate 1-dim arrays + boolArray = getBooleanEnumeration(0, nSizePrimitives); + byteArray = getByteEnumeration(1, nSizePrimitives + 1); + // charArray = getCharEnumeration(2, nSizePrimitives + 2); + shortArray = getShortEnumeration(3, nSizePrimitives + 3); + intArray = getIntEnumeration(4, nSizePrimitives + 4); + longArray = getLongEnumeration(5, nSizePrimitives + 5); + floatArray = getFloatEnumeration(6, nSizePrimitives + 6); + doubleArray = getDoubleEnumeration(7, nSizePrimitives + 7); + + // allocate n-dim arrays -- N.B. for simplicity the dimension/low-level backing size is const + + nDimensions = new int[] { 2, 3, 2 }; + final int nMultiDim = nDimensions[0] * nDimensions[1] * nDimensions[2]; + boolNdimArray = getBooleanEnumeration(0, nMultiDim); + byteNdimArray = getByteEnumeration(1, nMultiDim + 1); + // charNdimArray = getCharEnumeration(2, nMultiDim + 2); + shortNdimArray = getShortEnumeration(3, nMultiDim + 3); + intNdimArray = getIntEnumeration(4, nMultiDim + 4); + longNdimArray = getLongEnumeration(5, nMultiDim + 5); + floatNdimArray = getFloatEnumeration(6, nMultiDim + 6); + doubleNdimArray = getDoubleEnumeration(7, nMultiDim + 7); + } + + if (nSizeString >= 0) { + stringArray = new String[nSizeString]; + for (int i = 0; i < nSizeString; ++i) { + stringArray[i] = string1; + } + } + } + + public static boolean isCmwCompatibilityMode() { + return cmwCompatibilityMode; + } + + public static void setCmwCompatibilityMode(final boolean cmwCompatibilityMode) { + TestDataClass.cmwCompatibilityMode = cmwCompatibilityMode; + } + + private static boolean[] getBooleanEnumeration(final int from, final int to) { + final boolean[] ret = new boolean[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = i % 2 == 0; + } + return ret; + } + + private static byte[] getByteEnumeration(final int from, final int to) { + final byte[] ret = new byte[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = (byte) i; + } + return ret; + } + + private static char[] getCharEnumeration(final int from, final int to) { + final char[] ret = new char[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = (char) i; + } + return ret; + } + + private static double[] getDoubleEnumeration(final int from, final int to) { + final double[] ret = new double[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = i / 10f; + } + return ret; + } + + private static float[] getFloatEnumeration(final int from, final int to) { + final float[] ret = new float[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = i / 10f; + } + return ret; + } + + private static int[] getIntEnumeration(final int from, final int to) { + final int[] ret = new int[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = i; + } + return ret; + } + + private static long[] getLongEnumeration(final int from, final int to) { + final long[] ret = new long[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = i; + } + return ret; + } + + private static short[] getShortEnumeration(final int from, final int to) { + final short[] ret = new short[to - from]; + for (int i = from; i < to; i++) { + ret[i - from] = (short) i; + } + return ret; + } +} diff --git a/serialiser/src/test/resources/simplelogger.properties b/serialiser/src/test/resources/simplelogger.properties new file mode 100644 index 00000000..a01ef764 --- /dev/null +++ b/serialiser/src/test/resources/simplelogger.properties @@ -0,0 +1,50 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=debug + +# The output target which can be the path to a file, or the special values "System.out" and "System.err". +# Default is "System.err". +org.slf4j.simpleLogger.logFile=System.out + +# If the output target is set to "System.out" or "System.err" (see preceding entry), by default, +# logs will be output to the latest value referenced by System.out/err variables. +# By setting this parameter to true, the output stream will be cached, i.e. assigned once at initialization +# time and re-used independently of the current value referenced by System.out/err. +org.slf4j.simpleLogger.cacheOutputStream=true + +# Logging detail level for a SimpleLogger instance named "a.b.c". Right-side value must be one of +# "trace", "debug", "info", "warn", "error" or "off". When a SimpleLogger named "a.b.c" is initialized, +# its level is assigned from this property. If unspecified, the level of nearest parent logger will be used, +# and if none is set, then the value specified by org.slf4j.simpleLogger.defaultLogLevel will be used. +org.slf4j.simpleLogger.log.de.gsi.* + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +# org.slf4j.simpleLogger.showThreadName=false + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +# org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/server-rest/pom.xml b/server-rest/pom.xml new file mode 100644 index 00000000..99a8d51a --- /dev/null +++ b/server-rest/pom.xml @@ -0,0 +1,122 @@ + + + 4.0.0 + + + io.opencmw + opencmw + ${revision}${sha1}${changelist} + ../pom.xml + + + server-rest + + + OpenCMW RESTful plugin extension to micro-service implementation. + + + + + io.opencmw + server + ${revision}${sha1}${changelist} + + + de.gsi.dataset + chartfx-dataset + ${version.chartfx} + test + + + + + io.javalin + javalin + ${version.javalin} + + + ch.qos.logback + logback-classic + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + + + cc.vileda + kotlin-openapi3-dsl + + + io.github.classgraph + classgraph + + + com.fasterxml.jackson.core + jackson-databind + + + + + io.javalin + javalin-openapi + ${version.javalin} + + + + org.eclipse.jetty.http2 + http2-server + ${version.jetty} + + + org.eclipse.jetty + jetty-alpn-conscrypt-server + ${version.jetty} + + + + + org.apache.velocity + velocity-engine-core + ${version.velocity} + + + org.mindrot + jbcrypt + 0.4 + + + + + com.jsoniter + jsoniter + 0.9.23 + + + io.micrometer + micrometer-core + ${version.micrometer} + + + io.micrometer + micrometer-registry-prometheus + ${version.micrometer} + + + + + com.squareup.okhttp3 + okhttp + ${version.okHttp3} + test + + + com.squareup.okhttp3 + okhttp-sse + ${version.okHttp3} + test + + + + \ No newline at end of file diff --git a/server-rest/src/main/java/io/opencmw/server/rest/MajordomoRestPlugin.java b/server-rest/src/main/java/io/opencmw/server/rest/MajordomoRestPlugin.java new file mode 100644 index 00000000..c9ff4104 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/MajordomoRestPlugin.java @@ -0,0 +1,491 @@ +package io.opencmw.server.rest; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static io.javalin.apibuilder.ApiBuilder.get; +import static io.javalin.apibuilder.ApiBuilder.post; +import static io.javalin.plugin.openapi.dsl.DocumentedContentKt.anyOf; +import static io.javalin.plugin.openapi.dsl.DocumentedContentKt.documentedContent; +import static io.opencmw.OpenCmwProtocol.Command.GET_REQUEST; +import static io.opencmw.OpenCmwProtocol.Command.READY; +import static io.opencmw.OpenCmwProtocol.Command.SET_REQUEST; +import static io.opencmw.OpenCmwProtocol.Command.UNKNOWN; +import static io.opencmw.OpenCmwProtocol.EMPTY_FRAME; +import static io.opencmw.OpenCmwProtocol.MdpMessage; +import static io.opencmw.OpenCmwProtocol.MdpMessage.receive; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_CLIENT; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; +import static io.opencmw.server.MajordomoBroker.INTERNAL_ADDRESS_PUBLISHER; +import static io.opencmw.server.MmiServiceHelper.INTERNAL_SERVICE_NAMES; +import static io.opencmw.server.MmiServiceHelper.INTERNAL_SERVICE_OPENAPI; +import static io.opencmw.server.rest.RestServer.prefixPath; +import static io.opencmw.server.rest.util.CombinedHandler.SseState.CONNECTED; +import static io.opencmw.server.rest.util.CombinedHandler.SseState.DISCONNECTED; + +import java.lang.reflect.ParameterizedType; +import java.net.ProtocolException; +import java.net.URI; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.function.Predicate; + +import javax.validation.constraints.NotNull; + +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jetty.util.BlockingArrayQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +import io.javalin.apibuilder.ApiBuilder; +import io.javalin.core.security.Role; +import io.javalin.http.BadRequestResponse; +import io.javalin.http.Context; +import io.javalin.http.Handler; +import io.javalin.http.sse.SseClient; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import io.opencmw.MimeType; +import io.opencmw.OpenCmwProtocol; +import io.opencmw.QueryParameterParser; +import io.opencmw.rbac.RbacRole; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.serialiser.spi.ClassFieldDescription; +import io.opencmw.serialiser.utils.ClassUtils; +import io.opencmw.server.BasicMdpWorker; +import io.opencmw.server.MajordomoWorker; +import io.opencmw.server.rest.util.CombinedHandler; +import io.opencmw.server.rest.util.MessageBundle; +import io.opencmw.utils.CustomFuture; + +import com.jsoniter.output.JsonStream; + +/** + * Majordomo Broker REST/HTTP plugin. + * + * This opens two http ports and converts and forwards incoming request to the OpenCMW protocol and provides + * some basic admin functionality + * + *

+ * Server parameter can be controlled via the following system properties: + *

    + *
  • restServerHostName: host name or IP address the server should bind to + *
  • restServerPort: the HTTP port + *
  • restServerPort2: the HTTP/2 port (encrypted) + *
  • restKeyStore: the path to the file containing the key store for the encryption + *
  • restKeyStorePassword: the path to the file containing the key store for the encryption + *
  • restUserPasswordStore: the path to the file containing the user passwords and roles encryption + *
+ * @see RestServer for more details regarding the RESTful specific aspects + * + * @author rstein + */ +@MetaInfo(description = "Majordomo Broker REST/HTTP plugin.

" + + " This opens two http ports and converts and forwards incoming request to the OpenCMW protocol and provides
" + + " some basic admin functionality
", + unit = "MajordomoRestPlugin") +@SuppressWarnings({ "PMD.ExcessiveImports", "PMD.TooManyStaticImports", "PMD.DoNotUseThreads" }) // makes the code more readable/shorter lines +public class MajordomoRestPlugin extends BasicMdpWorker { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoRestPlugin.class); + private static final byte[] RBAC = {}; // TODO: implement RBAC between Majordomo and Worker + private static final String TEMPLATE_EMBEDDED_HTML = "/velocity/property/defaultTextPropertyLayout.vm"; + private static final String TEMPLATE_BAD_REQUEST = "/velocity/errors/badRequest.vm"; + private static final AtomicLong REQUEST_COUNTER = new AtomicLong(); + protected final ZMQ.Socket subSocket; + protected final Map subscriptionCount = new ConcurrentHashMap<>(); + protected static final byte[] REST_SUB_ID = "REST_SUBSCRIPTION".getBytes(UTF_8); + protected final ConcurrentMap registeredEndpoints = new ConcurrentHashMap<>(); + private final BlockingArrayQueue requestQueue = new BlockingArrayQueue<>(); + private final ConcurrentMap> requestReplies = new ConcurrentHashMap<>(); + private final BiConsumer newSseClientHandler; + + public MajordomoRestPlugin(ZContext ctx, final String serverDescription, String httpAddress, final RbacRole... rbacRoles) { + super(ctx, MajordomoRestPlugin.class.getSimpleName(), rbacRoles); + assert (httpAddress != null); + RestServer.setName(Objects.requireNonNullElse(serverDescription, MajordomoRestPlugin.class.getName())); + subSocket = ctx.createSocket(SocketType.SUB); + subSocket.setHWM(0); + subSocket.connect(INTERNAL_ADDRESS_PUBLISHER); + subSocket.subscribe(INTERNAL_SERVICE_NAMES); + subscriptionCount.computeIfAbsent(INTERNAL_SERVICE_NAMES, s -> new AtomicInteger()).incrementAndGet(); + + newSseClientHandler = (client, state) -> { + final String queryString = client.ctx.queryString() == null ? "" : ("?" + client.ctx.queryString()); + final String subService = StringUtils.stripEnd(StringUtils.stripStart(client.ctx.path(), "/"), "/") + queryString; + LOGGER.atDebug().addArgument(state).addArgument(subService).addArgument(subscriptionCount.computeIfAbsent(subService, s -> new AtomicInteger()).get()).log("RestPlugin {} to '{}' - existing subscriber count: {}"); + if (state == CONNECTED && subscriptionCount.computeIfAbsent(subService, s -> new AtomicInteger()).incrementAndGet() == 1) { + subSocket.subscribe(subService); + } + if (state == DISCONNECTED && subscriptionCount.computeIfAbsent(subService, s -> new AtomicInteger()).decrementAndGet() <= 0) { + subSocket.unsubscribe(subService); + subscriptionCount.remove(subService); + } + }; + + // add default root - here: redirect to mmi.service + RestServer.getInstance().get("/", restCtx -> restCtx.redirect("/mmi.service"), RestServer.getDefaultRole()); + + registerHandler(getDefaultRequestHandler()); // NOPMD - one-time call OK + + LOGGER.atInfo().addArgument(MajordomoRestPlugin.class.getName()).addArgument(RestServer.getPublicURI()).log("{} started on address: {}"); + } + + @Override + public boolean notify(@NotNull final MdpMessage notifyMessage) { + assert notifyMessage != null : "notify message must not be null"; + notifyRaw(notifyMessage); + return false; + } + + @Override + public synchronized void start() { // NOPMD 'synchronized' comes from JDK class definition + final Thread dispatcher = new Thread(getDispatcherTask()); + dispatcher.setDaemon(true); + dispatcher.setName(MajordomoRestPlugin.class.getSimpleName() + "Dispatcher"); + dispatcher.start(); + + final Thread serviceListener = new Thread(getServiceSubscriptionTask()); + serviceListener.setDaemon(true); + serviceListener.setName(MajordomoRestPlugin.class.getSimpleName() + "Subscriptions"); + serviceListener.start(); + + // send subscription request for new service added notifications + super.start(); + + // perform initial get request + String services = "(uninitialised)"; + final CustomFuture reply = dispatchRequest(new MdpMessage(null, PROT_CLIENT, GET_REQUEST, INTERNAL_SERVICE_NAMES.getBytes(UTF_8), EMPTY_FRAME, URI.create(INTERNAL_SERVICE_NAMES), EMPTY_FRAME, "", RBAC), true); + try { + final MdpMessage msg = reply.get(); + services = msg.data == null ? "" : new String(msg.data, UTF_8); + Arrays.stream(StringUtils.split(services, ",:;")).forEach(this::registerEndPoint); + } catch (final Exception e) { // NOPMD -- erroneous worker replies shall not stop the broker + LOGGER.atError().setCause(e).addArgument(services).log("could not perform initial registering of endpoints {}"); + } + } + + protected static OpenCmwProtocol.Command getCommand(@NotNull final Context restCtx) { + switch (restCtx.method()) { + case "GET": + return GET_REQUEST; + case "POST": + return SET_REQUEST; + default: + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(restCtx.req).log("unknown request: {}"); + } + return UNKNOWN; + } + } + + protected RequestHandler getDefaultRequestHandler() { + return handler -> { + switch (handler.req.command) { + case PARTIAL: + case FINAL: + if (handler.req.clientRequestID.length == 0 || Arrays.equals(REST_SUB_ID, handler.req.clientRequestID)) { + handler.rep = null; // NOPMD needs to be 'null' to suppress message being further processed + break; + } + final String clientRequestID = new String(handler.req.clientRequestID, UTF_8); + final CustomFuture replyFuture = requestReplies.remove(clientRequestID); + if (replyFuture == null) { + LOGGER.atWarn().addArgument(clientRequestID).addArgument(handler.req).log("could not match clientRequestID '{}' to Future. msg was: {}"); + return; + } + if (handler.req.errors == null || handler.req.errors.isBlank()) { + replyFuture.setReply(handler.req); + } else { + // exception occurred - forward it + replyFuture.setException(new ProtocolException(handler.req.errors)); + } + handler.rep = null; // NOPMD needs to be 'null' to suppress message being further processed + return; + case W_NOTIFY: + final String serviceName = handler.req.getSenderName(); + final String topicName = handler.req.topic.toString(); + final long eventTimeStamp = System.currentTimeMillis(); + final String notifyMessage = "new '" + topicName + "' @" + eventTimeStamp; + final Queue sseClients = RestServer.getEventClients(serviceName); + sseClients.forEach((final SseClient client) -> client.sendEvent(notifyMessage)); + return; + case GET_REQUEST: + case SET_REQUEST: + case DISCONNECT: + case READY: + case SUBSCRIBE: + case UNSUBSCRIBE: + case W_HEARTBEAT: + case UNKNOWN: + default: + break; + } + }; + } + + protected Runnable getDispatcherTask() { + return () -> { + final Queue notifyCopy = new ArrayDeque<>(); + while (runSocketHandlerLoop.get() && !Thread.interrupted()) { + synchronized (requestQueue) { + try { + requestQueue.wait(); + if (!requestQueue.isEmpty()) { + notifyCopy.addAll(requestQueue); + requestQueue.clear(); + } + } catch (InterruptedException e) { + LOGGER.atWarn().setCause(e).log("Interrupted!"); + // restore interrupted state... + Thread.currentThread().interrupt(); + } + } + if (notifyCopy.isEmpty()) { + continue; + } + notifyCopy.forEach(this::notify); + notifyCopy.clear(); + } + }; + } + + protected Runnable getServiceSubscriptionTask() { // NOSONAR NOPMD - complexity is acceptable + return () -> { + try (ZMQ.Poller subPoller = ctx.createPoller(1)) { + subPoller.register(subSocket, ZMQ.Poller.POLLIN); + while (runSocketHandlerLoop.get() && !Thread.interrupted() && subPoller.poll(TimeUnit.MILLISECONDS.toMillis(100)) != -1) { + // handle message from or to broker + boolean dataReceived = true; + while (dataReceived) { + dataReceived = false; + // handle subscription message from or to broker + final MdpMessage brokerMsg = receive(subSocket, true); + if (brokerMsg != null) { + dataReceived = true; + liveness = HEARTBEAT_LIVENESS; + + // handle subscription message + if (brokerMsg.data != null && brokerMsg.getServiceName().startsWith(INTERNAL_SERVICE_NAMES)) { + registerEndPoint(new String(brokerMsg.data, UTF_8)); // NOPMD in-loop instantiation necessary + } + notifySubscribedClients(brokerMsg.topic); + } + } + } + } + }; + } + + @Override + protected void reconnectToBroker() { + super.reconnectToBroker(); + final byte[] classNameByte = this.getClass().getName().getBytes(UTF_8); // used for OpenAPI purposes + new MdpMessage(null, PROT_WORKER, READY, serviceBytes, EMPTY_FRAME, RestServer.getPublicURI(), classNameByte, "", RBAC).send(workerSocket); + new MdpMessage(null, PROT_WORKER, READY, serviceBytes, EMPTY_FRAME, RestServer.getLocalURI(), classNameByte, "", RBAC).send(workerSocket); + } + + protected void registerEndPoint(final String endpoint) { + synchronized (registeredEndpoints) { + // needs to be synchronised since Javalin get(..), put(..) seem to be not thread safe (usually initialised during startup) + registeredEndpoints.computeIfAbsent(endpoint, ep -> { + final MdpMessage requestMsg = new MdpMessage(null, PROT_CLIENT, GET_REQUEST, INTERNAL_SERVICE_OPENAPI.getBytes(UTF_8), EMPTY_FRAME, URI.create(INTERNAL_SERVICE_OPENAPI), ep.getBytes(UTF_8), "", RBAC); + final CustomFuture openApiReply = dispatchRequest(requestMsg, true); + try { + final MdpMessage serviceOpenApiData = openApiReply.get(); + if (!serviceOpenApiData.errors.isBlank()) { + LOGGER.atWarn().addArgument(ep).addArgument(serviceOpenApiData).log("received erroneous message for service '{}': {}"); + return null; + } + final String handlerClassName = new String(serviceOpenApiData.data, UTF_8); + + OpenApiDocumentation openApi = getOpenApiDocumentation(handlerClassName); + + final Set accessRoles = RestServer.getDefaultRole(); + RestServer.getInstance().routes(() -> { + ApiBuilder.before(ep, restCtx -> { + // for some strange reason this needs to be executed to be able to read 'restCtx.formParamMap()' + if ("POST".equals(restCtx.method())) { + final Map> map = restCtx.formParamMap(); + if (map.size() == 0) { + LOGGER.atDebug().addArgument(restCtx.req.getPathInfo()).log("{} called without form data"); + } + } + }); + post(ep + "*", OpenApiBuilder.documented(openApi, getDefaultServiceRestHandler(ep)), accessRoles); + get(ep + "*", OpenApiBuilder.documented(openApi, getDefaultServiceRestHandler(ep)), accessRoles); + }); + + return openApi; + } catch (final Exception e) { // NOPMD -- erroneous worker replies shall not stop the broker + LOGGER.atError().setCause(e).addArgument(ep).log("could not register endpoint {}"); + } + return null; + }); + } + } + + @org.jetbrains.annotations.NotNull + private OpenApiDocumentation getOpenApiDocumentation(final String handlerClassName) { + OpenApiDocumentation openApi = OpenApiBuilder.document(); + try { + final Class clazz = Class.forName(handlerClassName); + final ClassFieldDescription fieldDescription = ClassUtils.getFieldDescription(clazz); + openApi.operation(openApiOperation -> { + openApiOperation.description(fieldDescription.getFieldDescription() + " - " + handlerClassName); + openApiOperation.operationId("myOperationId"); + openApiOperation.summary(fieldDescription.getFieldUnit()); + openApiOperation.deprecated(false); + openApiOperation.addTagsItem("user"); + }); + + if (MajordomoWorker.class.isAssignableFrom(clazz)) { + // class is a MajordomoWorker derivative + final ParameterizedType genericSuperClass = (ParameterizedType) clazz.getGenericSuperclass(); + final Class ctxClass = (Class) genericSuperClass.getActualTypeArguments()[0]; + final Class inClass = (Class) genericSuperClass.getActualTypeArguments()[1]; + final Class outClass = (Class) genericSuperClass.getActualTypeArguments()[2]; + + final ClassFieldDescription ctxFilter = ClassUtils.getFieldDescription(ctxClass); + for (FieldDescription field : ctxFilter.getChildren()) { + ClassFieldDescription classField = (ClassFieldDescription) field; + openApi.queryParam(classField.getFieldName(), (Class) classField.getType()); + openApi.formParam(classField.getFieldName(), (Class) classField.getType(), false); // find definition for required or not + } + + openApi.body(anyOf(documentedContent(outClass), documentedContent(inClass))); + openApi.body(outClass).json("200", outClass); // JSON definition + openApi.html("200").result("demo output"); // HTML definition + + //TODO: continue here -- work in progress + } + + } catch (Exception e) { // NOPMD + LOGGER.atWarn().setCause(e).addArgument(handlerClassName).log("could not find class definition for {}"); + } + return openApi; + } + + private CustomFuture dispatchRequest(final MdpMessage requestMsg, boolean expectReply) { + final String requestID = MajordomoRestPlugin.class.getSimpleName() + "#" + REQUEST_COUNTER.getAndIncrement(); + requestMsg.clientRequestID = requestID.getBytes(UTF_8); + + if (expectReply) { + requestMsg.clientRequestID = requestID.getBytes(UTF_8); + } else { + requestMsg.clientRequestID = REST_SUB_ID; + } + CustomFuture reply = new CustomFuture<>(); + final Object ret = requestReplies.put(requestID, reply); + if (ret != null) { + LOGGER.atWarn().addArgument(requestID).addArgument(requestMsg.getServiceName()).log("duplicate request {} for service {}"); + } + + if (!requestQueue.offer(requestMsg)) { + throw new IllegalStateException("could not add MdpMessage to requestQueue: " + requestMsg); + } + synchronized (requestQueue) { + requestQueue.notifyAll(); + } + return reply; + } + + protected void notifySubscribedClients(final @NotNull URI topic) { + final String topicString = topic.toString(); + final String notifyPath = prefixPath(topic.getPath()); + // TODO: upgrade to path & query matching - for the time being only path @see also CombinedHandler + final Queue clients = RestServer.getEventClients(notifyPath); + final Predicate filter = c -> { + final String clientPath = StringUtils.stripEnd(c.ctx.path(), "/"); + return clientPath.length() >= notifyPath.length() && clientPath.startsWith(notifyPath); + }; + clients.stream().filter(filter).forEach(s -> s.sendEvent(topicString)); + } + + private Handler getDefaultServiceRestHandler(final String restHandler) { // NOSONAR NOPMD - complexity is acceptable + return new CombinedHandler(restCtx -> { + if (LOGGER.isTraceEnabled()) { + LOGGER.atTrace().addArgument(restHandler).addArgument(restCtx.path()).addArgument(restCtx.fullUrl()).log("restHandler {} for service {} - full: {}"); + } + + final String service = StringUtils.stripStart(Objects.requireNonNullElse(restCtx.path(), restHandler), "/"); + final MimeType acceptMimeType = MimeType.getEnum(restCtx.header(RestServer.HTML_ACCEPT)); + final Map parameterMap = restCtx.req.getParameterMap(); + final String[] mimeType = parameterMap.get("contentType"); + final URI topic = mimeType == null || mimeType.length == 0 ? RestServer.appendUri(URI.create(restCtx.fullUrl()), "contentType=" + acceptMimeType.toString()) : URI.create(restCtx.fullUrl()); + + OpenCmwProtocol.Command cmd = getCommand(restCtx); + final byte[] requestData; + if (cmd == SET_REQUEST) { + requestData = getFormDataAsJson(restCtx); + } else { + requestData = EMPTY_FRAME; + } + final MdpMessage requestMsg = new MdpMessage(null, PROT_CLIENT, cmd, service.getBytes(UTF_8), EMPTY_FRAME, topic, requestData, "", RBAC); + + CustomFuture reply = dispatchRequest(requestMsg, true); + try { + final MdpMessage replyMessage = reply.get(); //TODO: add max time-out -- only if not long-polling (to be checked) + final @NotNull MimeType replyMimeType = QueryParameterParser.getMimeType(replyMessage.topic.getQuery()); + switch (replyMimeType) { + case HTML: + case TEXT: + final String queryString = topic.getQuery() == null ? "" : ("?" + topic.getQuery()); + if (cmd == SET_REQUEST) { + final String path = restCtx.req.getRequestURI() + StringUtils.replace(queryString, "&noMenu", ""); + restCtx.redirect(path); + } else { + final boolean noMenu = queryString.contains("noMenu"); + Map dataMap = MessageBundle.baseModel(restCtx); + dataMap.put("textBody", new String(replyMessage.data, UTF_8)); + dataMap.put("noMenu", noMenu); + restCtx.render(TEMPLATE_EMBEDDED_HTML, dataMap); + } + break; + case BINARY: + default: + restCtx.contentType(replyMimeType.toString()); + restCtx.result(replyMessage.data); + + break; + } + + } catch (Exception e) { // NOPMD - exception is rethrown + switch (acceptMimeType) { + case HTML: + case TEXT: + Map dataMap = MessageBundle.baseModel(restCtx); + dataMap.put("service", restHandler); + dataMap.put("exceptionText", e); + restCtx.render(TEMPLATE_BAD_REQUEST, dataMap); + return; + default: + } + throw new BadRequestResponse(MajordomoRestPlugin.class.getName() + ": could not process service '" + service + "' - exception:\n" + e.getMessage()); // NOPMD original exception forwared within the text, BadRequestResponse does not support exception forwarding + } + }, newSseClientHandler); + } + + private byte[] getFormDataAsJson(final Context restCtx) { + final byte[] requestData; + final Map> formMap = restCtx.formParamMap(); + final HashMap requestMap = new HashMap<>(); + formMap.forEach((k, v) -> { + if (v.isEmpty()) { + requestMap.put(k, null); + } else { + requestMap.put(k, v.get(0)); + } + }); + final String formData = JsonStream.serialize(requestMap); + requestData = formData.getBytes(UTF_8); + return requestData; + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/RestCommonThreadPool.java b/server-rest/src/main/java/io/opencmw/server/rest/RestCommonThreadPool.java new file mode 100644 index 00000000..fc4de1f1 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/RestCommonThreadPool.java @@ -0,0 +1,69 @@ +package io.opencmw.server.rest; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +@SuppressWarnings("PMD.DoNotUseThreads") // purpose of this class +public final class RestCommonThreadPool implements ThreadFactory { + private static final int MAX_THREADS = getDefaultThreadCount(); + private static final int MAX_SCHEDULED_THREADS = getDefaultScheduledThreadCount(); + private static final ThreadFactory DEFAULT_FACTORY = Executors.defaultThreadFactory(); + private static final RestCommonThreadPool SELF = new RestCommonThreadPool(); + private static final ExecutorService COMMON_POOL = Executors.newFixedThreadPool(MAX_THREADS, SELF); + private static final ScheduledExecutorService SCHEDULED_POOL = Executors.newScheduledThreadPool(MAX_SCHEDULED_THREADS, SELF); + private static final AtomicInteger THREAD_COUNTER = new AtomicInteger(); + + private RestCommonThreadPool() { + // helper class + } + + @Override + public Thread newThread(final Runnable r) { + final Thread thread = DEFAULT_FACTORY.newThread(r); + THREAD_COUNTER.incrementAndGet(); + thread.setName("RestCommonThreadPool#" + THREAD_COUNTER.intValue()); + thread.setDaemon(true); + return thread; + } + + public static ExecutorService getCommonPool() { + return COMMON_POOL; + } + + public static ScheduledExecutorService getCommonScheduledPool() { + return SCHEDULED_POOL; + } + + public static RestCommonThreadPool getInstance() { + return SELF; + } + + public static int getNumbersOfThreads() { + return MAX_THREADS; + } + + private static int getDefaultScheduledThreadCount() { + int nthreads = 32; + try { + nthreads = Integer.parseInt(System.getProperty("restScheduledThreadCount", "32")); + } catch (final NumberFormatException e) { + // malformed number + } + + return Math.max(32, nthreads); + } + + private static int getDefaultThreadCount() { + int nthreads = 32; + try { + nthreads = Integer.parseInt(System.getProperty("restThreadCount", "64")); + } catch (final NumberFormatException e) { + // malformed number + } + + return Math.max(32, nthreads); + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/RestRole.java b/server-rest/src/main/java/io/opencmw/server/rest/RestRole.java new file mode 100644 index 00000000..f91e9595 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/RestRole.java @@ -0,0 +1,23 @@ +package io.opencmw.server.rest; + +import org.jetbrains.annotations.NotNull; + +import io.javalin.core.security.Role; +import io.opencmw.rbac.RbacRole; + +/** + * REST specific role adapter mapping of OpenCMW's RbacRole to Javalin's Role interface description + * @author rstein + */ +public class RestRole implements Role { + public final RbacRole rbacRole; + + public RestRole(@NotNull final RbacRole rbacRole) { + this.rbacRole = rbacRole; + } + + @Override + public String toString() { + return rbacRole.toString(); + } +} \ No newline at end of file diff --git a/server-rest/src/main/java/io/opencmw/server/rest/RestServer.java b/server-rest/src/main/java/io/opencmw/server/rest/RestServer.java new file mode 100644 index 00000000..5bf2bbde --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/RestServer.java @@ -0,0 +1,497 @@ +package io.opencmw.server.rest; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import javax.servlet.ServletOutputStream; + +import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory; +import org.eclipse.jetty.http2.HTTP2Cipher; +import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javalin.Javalin; +import io.javalin.apibuilder.ApiBuilder; +import io.javalin.core.compression.Gzip; +import io.javalin.core.event.HandlerMetaInfo; +import io.javalin.core.security.Role; +import io.javalin.core.util.Header; +import io.javalin.core.util.RouteOverviewPlugin; +import io.javalin.http.Context; +import io.javalin.http.sse.SseClient; +import io.javalin.http.util.RateLimit; +import io.javalin.plugin.json.JavalinJson; +import io.javalin.plugin.metrics.MicrometerPlugin; +import io.javalin.plugin.openapi.OpenApiOptions; +import io.javalin.plugin.openapi.OpenApiPlugin; +import io.javalin.plugin.openapi.annotations.HttpMethod; +import io.javalin.plugin.openapi.ui.ReDocOptions; +import io.javalin.plugin.openapi.ui.SwaggerOptions; +import io.opencmw.MimeType; +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacRole; +import io.opencmw.server.rest.admin.RestServerAdmin; +import io.opencmw.server.rest.login.LoginController; +import io.opencmw.server.rest.user.RestUserHandler; +import io.opencmw.server.rest.user.RestUserHandlerImpl; +import io.opencmw.server.rest.util.MessageBundle; +import io.swagger.v3.oas.models.info.Info; + +import com.jsoniter.JsonIterator; +import com.jsoniter.output.JsonStream; + +/** + * Small RESTful server helper class. + * + *

+ * The Javalin framework is being used internally: https://javalin.io/ + * + * The primary purposes of this utility class is to provide + *

    + *
  • some convenience methods, default configuration (in particular relating to SSL and HTTP/2), and + *
  • to wrap the primary REST server implementation in view of back-end server upgrades or changing API. + *
  • to provide every GET route also with an long-polling and SSE listener/data-retrieval management. + *
+ * + *

+ * Server parameter can be controlled via the following system properties: + *

    + *
  • restServerHostName: host name or IP address the server should bind to + *
  • restServerPort: the HTTP port + *
  • restServerPort2: the HTTP/2 port (encrypted) + *
  • restKeyStore: the path to the file containing the key store for the encryption + *
  • restKeyStorePassword: the path to the file containing the key store for the encryption + *
  • restUserPasswordStore: the path to the file containing the user passwords and roles encryption + *
+ * some design choices: minimise exposing Javalin API outside this class, no usage of UI specific classes (ie. JavaFX) + * + * @author rstein + */ +@SuppressWarnings("PMD.ExcessiveImports") +public final class RestServer { // NOPMD -- nomen est omen + public static final String TAG_REST_SERVER_HOST_NAME = "restServerHostName"; + public static final String TAG_REST_SERVER_PORT = "restServerPort"; + public static final String TAG_REST_SERVER_PORT2 = "restServerPort2"; + public static final String REST_KEY_STORE = "restKeyStore"; + public static final String REST_KEY_STORE_PASSWORD = "restKeyStorePassword"; + // some HTML constants + public static final String HTML_ACCEPT = "accept"; + private static final Logger LOGGER = LoggerFactory.getLogger(RestServer.class); + private static final String DEFAULT_HOST_NAME = "0"; + private static final int DEFAULT_PORT = 8080; + private static final int DEFAULT_PORT2 = 8443; + private static final String REST_PROTOCOL = "protocol"; + + private static final String TEMPLATE_UNAUTHORISED = "/velocity/errors/unauthorised.vm"; + private static final String TEMPLATE_ACCESS_DENIED = "/velocity/errors/accessDenied.vm"; + private static final String TEMPLATE_NOT_FOUND = "/velocity/errors/notFound.vm"; + private static final String TEMPLATE_BAD_REQUEST = "/velocity/errors/badRequest.vm"; + private static final ConcurrentMap> EVENT_LISTENER_SSE = new ConcurrentHashMap<>(); + private static final List ENDPOINTS = new ArrayList<>(); + private static final Consumer ENDPOINT_ADDED_HANDLER = ENDPOINTS::add; + private static Javalin instance; + private static MimeType defaultProtocol = MimeType.HTML; + private static RestUserHandler userHandler = new RestUserHandlerImpl(BasicRbacRole.NULL); // include basic Rbac role definition + private static String serverName = "Undefined REST Server"; + + private RestServer() { + // this is a utility class + } + + public static void addLongPollingCookie(final Context ctx, final String key, final long lastUpdateMillies) { + // N.B. this is a workaround since javax.servlet.http.Cookie does not support the SameSite cookie field. + // workaround inspired by: https://github.com/tipsy/javalin/issues/780 + final String cookieComment = "stores the servcer-side time stamp of the last valid update (required for long-polling)"; + final String cookie = key + "=" + lastUpdateMillies + "; Comment=\"" + cookieComment + "\"; Expires=-1; SameSite=Strict;"; + ctx.res.addHeader("Set-Cookie", cookie); + } + + public static URI appendUri(URI oldUri, String appendQuery) throws URISyntaxException { + return new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(), + oldUri.getQuery() == null ? appendQuery : oldUri.getQuery() + "&" + appendQuery, oldUri.getFragment()); + } + + /** + * guards this end point and returns HTTP error response if predefined rate limit is exceeded + * + * @param ctx end point context handler + * @param numRequests number of calls + * @param timeUnit time base reference + */ + public static void applyRateLimit(final Context ctx, final int numRequests, final TimeUnit timeUnit) { + new RateLimit(ctx).requestPerTimeUnit(numRequests, timeUnit); // + } + + public static MimeType getDefaultProtocol() { + return defaultProtocol; + } + + public static Set getDefaultRole() { + return Collections.singleton(new RestRole(BasicRbacRole.ANYONE)); + } + + public static List getEndpoints() { + return ENDPOINTS; + } + + public static Queue getEventClients(@NotNull final String endpointName) { + if (endpointName.isEmpty()) { + throw new IllegalArgumentException("endpointNmae must not be empty"); + } + + final String fullEndPointName = prefixPath(endpointName); + return EVENT_LISTENER_SSE.computeIfAbsent(fullEndPointName, key -> new ConcurrentLinkedQueue<>()); + } + + public static ConcurrentMap> getEventClientMap() { + return EVENT_LISTENER_SSE; + } + + public static String getHostName() { + return System.getProperty(TAG_REST_SERVER_HOST_NAME, DEFAULT_HOST_NAME); + } + + public static int getHostPort() { + final String property = System.getProperty(TAG_REST_SERVER_PORT, Integer.toString(DEFAULT_PORT)); + try { + return Integer.parseInt(property); + } catch (final NumberFormatException e) { + LOGGER.atError().addArgument(TAG_REST_SERVER_PORT).addArgument(property).addArgument(DEFAULT_PORT).log("could not parse {}='{}' return default port {}"); + return DEFAULT_PORT; + } + } + + public static int getHostPort2() { + final String property = System.getProperty(TAG_REST_SERVER_PORT2, Integer.toString(DEFAULT_PORT2)); + try { + return Integer.parseInt(property); + } catch (final NumberFormatException e) { + LOGGER.atError().addArgument(TAG_REST_SERVER_PORT2).addArgument(property).addArgument(DEFAULT_PORT2).log("could not parse {}='{}' return default port {}"); + return DEFAULT_PORT2; + } + } + + public static Javalin getInstance() { + if (instance == null) { + startRestServer(); + } + return instance; + } + + public static URI getLocalURI() { + try { + return new URI("http://localhost:" + getHostPort()); + } catch (final URISyntaxException e) { + LOGGER.atError().setCause(e).log("getLocalURL()"); + } + return null; + } + + public static String getName() { + return serverName; + } + + public static URI getPublicURI() { + final String ip = getLocalHostName(); + try (DatagramSocket socket = new DatagramSocket()) { + return new URI("https://" + ip + ":" + getHostPort2()); + } catch (final URISyntaxException | SocketException e) { + LOGGER.atError().setCause(e).log("getPublicURL()"); + } + return null; + } + + public static MimeType getRequestedMimeProtocol(final Context ctx, final MimeType... defaultProtocol) { + return MimeType.getEnum(getRequestedProtocol(ctx, defaultProtocol.length == 0 ? getDefaultProtocol().toString() : defaultProtocol[0].toString())); + } + + public static String getRequestedProtocol(final Context ctx, final String... defaultProtocol) { + String protocol = defaultProtocol.length == 0 ? getDefaultProtocol().toString() : defaultProtocol[0]; + String protocolHeader = ctx.header(Header.ACCEPT); + String protocolQuery = ctx.queryParam(REST_PROTOCOL); + + if (protocolHeader != null && !protocolHeader.isBlank()) { + protocol = protocolHeader; + } + if (protocolQuery != null && !protocolQuery.isBlank()) { + protocol = protocolQuery; + } + + return protocol; + } + + public static Set getSessionCurrentRoles(final Context ctx) { + return LoginController.getSessionCurrentRoles(ctx); + } + + public static String getSessionCurrentUser(final Context ctx) { + return LoginController.getSessionCurrentUser(ctx); + } + + public static String getSessionLocale(final Context ctx) { + return LoginController.getSessionLocale(ctx); + } + + public static RestUserHandler getUserHandler() { + return userHandler; + } + + public static String prefixPath(@NotNull final String path) { + return ApiBuilder.prefixPath(path); + } + + public static void setDefaultProtocol(MimeType defaultProtocol) { + RestServer.defaultProtocol = defaultProtocol; + } + + public static void setName(final String serverName) { + RestServer.serverName = serverName; + } + + /** + * Sets a new user handler. + * + * N.B: This will issue a warning to remind system admins or security-minded people + * that the default implementation may have been replaced with a better/worse/different implementation (e.g. based on + * LDAP or another data base) + * + * @param newUserHandler the new implementation + */ + public static void setUserHandler(final RestUserHandler newUserHandler) { + LOGGER.atWarn().addArgument(newUserHandler.getClass().getCanonicalName()).log("replacing default user handler with '{}'"); + userHandler = newUserHandler; + } + + public static void startRestServer() { + JavalinJson.setFromJsonMapper(JsonIterator::deserialize); + JavalinJson.setToJsonMapper(JsonStream::serialize); + instance = Javalin.create(config -> { + config.enableCorsForAllOrigins(); + config.addStaticFiles("/public"); + config.showJavalinBanner = false; + config.defaultContentType = getDefaultProtocol().toString(); + config.compressionStrategy(null, new Gzip(6)); + config.server(RestServer::createHttp2Server); + // show all routes on specified path + config.registerPlugin(new RouteOverviewPlugin("/admin/endpoints", Collections.singleton(new RestRole(BasicRbacRole.ADMIN)))); + config.registerPlugin(new MicrometerPlugin()); + config.sessionHandler(getCustomSessionHandlerSupplier()); + // add OpenAPI + config.registerPlugin(new OpenApiPlugin(getOpenApiOptions())); + }) + .events(event -> event.handlerAdded(ENDPOINT_ADDED_HANDLER)); + instance.start(); + + // add login management + LoginController.register(); + + // add basic RestServer admin interface + RestServerAdmin.register(); + + // some default error mappings + instance.error(400, ctx -> ctx.render(TEMPLATE_BAD_REQUEST, MessageBundle.baseModel(ctx))); + instance.error(401, ctx -> ctx.render(TEMPLATE_UNAUTHORISED, MessageBundle.baseModel(ctx))); + instance.error(403, ctx -> ctx.render(TEMPLATE_ACCESS_DENIED, MessageBundle.baseModel(ctx))); + instance.error(404, ctx -> ctx.render(TEMPLATE_NOT_FOUND, MessageBundle.baseModel(ctx))); + } + + public static void startRestServer(final int hostPort, final int hostPort2) { + System.setProperty(TAG_REST_SERVER_PORT, Integer.toString(hostPort)); + System.setProperty(TAG_REST_SERVER_PORT2, Integer.toString(hostPort2)); + startRestServer(); + } + + public static void startRestServer(final String hostName, final int hostPort, final int hostPort2) { + System.setProperty(TAG_REST_SERVER_HOST_NAME, hostName); + System.setProperty(TAG_REST_SERVER_PORT, Integer.toString(hostPort)); + System.setProperty(TAG_REST_SERVER_PORT2, Integer.toString(hostPort2)); + startRestServer(); + } + + public static void stopRestServer() { + if (Objects.requireNonNull(RestServer.getInstance().server()).server().isRunning()) { + RestServer.getInstance().stop(); + } + } + + /** + * Suppresses caching for this end point + * + * @param ctx end point context handler + */ + public static void suppressCaching(final Context ctx) { + // for for HTTP 1.1 + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control + ctx.res.addHeader("Cache-Control", "no-store"); + + // for HTTP 1.0 + ctx.res.addHeader("Pragma", "no-cache"); + + // for proxies: may need to check an appropriate value + ctx.res.addHeader("Expires", "0"); + } + + public static void writeBytesToContext(@NotNull final Context ctx, final byte[] bytes, final int nSize) { + // based on the suggestions at https://github.com/tipsy/javalin/issues/910 + try (ServletOutputStream outputStream = ctx.res.getOutputStream()) { + outputStream.write(bytes, 0, nSize); + outputStream.flush(); + } catch (final IOException e) { + LOGGER.atError().setCause(e); + } + } + + private static Server createHttp2Server() { + final Server server = new Server(); + + // unencrypted HTTP 1 anchor + try (ServerConnector connector = new ServerConnector(server)) { + final String hostName = getHostName(); + final int hostPort = getHostPort(); + LOGGER.atInfo().addArgument(getLocalHostName()).log("local hostname = '{}'"); + LOGGER.atInfo().addArgument(hostName).addArgument(hostPort).log("create HTTP 1.x connector at 'http://{}:{}'"); + connector.setHost(hostName); + connector.setPort(hostPort); + server.addConnector(connector); + } + + // HTTP Configuration + final HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setSendServerVersion(false); + httpConfig.setSecureScheme("https"); + httpConfig.setSecurePort(getHostPort2()); + + // HTTPS Configuration + final HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig); + httpsConfig.addCustomizer(new SecureRequestCustomizer()); + + // HTTP/2 Connection Factory + final HTTP2ServerConnectionFactory h2 = new HTTP2ServerConnectionFactory(httpsConfig); + final ALPNServerConnectionFactory alpn = new ALPNServerConnectionFactory(); + alpn.setDefaultProtocol("h2"); + + // SSL Connection Factory + final SslConnectionFactory ssl = new SslConnectionFactory(createSslContextFactory(), alpn.getProtocol()); + + // HTTP/2 Connector + try (ServerConnector http2Connector = new ServerConnector(server, ssl, alpn, h2, new HttpConnectionFactory(httpsConfig))) { + final String hostName = getHostName(); + final int hostPort = getHostPort2(); + LOGGER.atInfo().addArgument(hostName).addArgument(hostPort).log("create HTTP/2 connector at 'http://{}:{}'"); + http2Connector.setHost(hostName); + http2Connector.setPort(hostPort); + server.addConnector(http2Connector); + } + + return server; + } + + private static SslContextFactory createSslContextFactory() { + final String keyStoreFile = System.getProperty(REST_KEY_STORE, null); // replace default with your real keystore + final String keyStorePwdFile = System.getProperty(REST_KEY_STORE_PASSWORD, null); // replace default with your real password + if (keyStoreFile == null || keyStorePwdFile == null) { + LOGGER.atInfo().addArgument(keyStoreFile).addArgument(keyStorePwdFile).log("using internal keyStore {} and/or keyStorePasswordFile {} -- PLEASE CHANGE FOR PRODUCTION -- THIS IS UNSAFE PRACTICE"); + } + LOGGER.atInfo().addArgument(keyStoreFile).log("using keyStore at '{}'"); + LOGGER.atInfo().addArgument(keyStorePwdFile).log("using keyStorePasswordFile at '{}'"); + + boolean readComplete = true; + String keyStorePwd = null; + KeyStore keyStore = null; + + // read keyStore password + try (BufferedReader br = keyStorePwdFile == null ? new BufferedReader(new InputStreamReader(RestServer.class.getResourceAsStream("/keystore.pwd"), UTF_8)) // + : Files.newBufferedReader(Paths.get(keyStorePwdFile), UTF_8)) { + keyStorePwd = br.readLine(); + } catch (final IOException e) { + readComplete = false; + LOGGER.atError().setCause(e).addArgument(keyStorePwdFile).log("error while reading key store password from '{}'"); + } + + if (readComplete && keyStorePwd != null) { + // read the actual keyStore + try (InputStream is = keyStoreFile == null ? RestServer.class.getResourceAsStream("/keystore.jks") // + : Files.newInputStream(Paths.get(keyStoreFile))) { + keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + keyStore.load(is, keyStorePwd.toCharArray()); + } catch (final NoSuchAlgorithmException | CertificateException | KeyStoreException | IOException e) { + readComplete = false; + LOGGER.atError().setCause(e).addArgument(keyStoreFile == null ? "internal" : keyStoreFile).log("error while reading key store from '{}'"); + } + } + + // SSL Context Factory for HTTPS and HTTP/2 + //noinspection deprecation + final SslContextFactory sslContextFactory = new SslContextFactory(true) {}; // trust all certificates + if (readComplete) { + sslContextFactory.setKeyStore(keyStore); + sslContextFactory.setKeyStorePassword(keyStorePwd); + } + sslContextFactory.setCipherComparator(HTTP2Cipher.COMPARATOR); + sslContextFactory.setProvider("Conscrypt"); + + return sslContextFactory; + } + + /** + * + * @return custom session handler that sets Jetty's JSESSIONID cookie to SameSite=strict + * + * N.B. to be used within Javalin's 'config.sessionHandler(getCustomSessionHandlerSupplier());' + */ + private static Supplier getCustomSessionHandlerSupplier() { + final SessionHandler sessionHandler = new SessionHandler(); + sessionHandler.getSessionCookieConfig().setHttpOnly(true); + sessionHandler.getSessionCookieConfig().setSecure(true); + sessionHandler.getSessionCookieConfig().setComment("__SAME_SITE_STRICT__"); + return () -> sessionHandler; + } + + private static String getLocalHostName() { + String ip; + try (DatagramSocket socket = new DatagramSocket()) { + socket.connect(InetAddress.getByName("8.8.8.8"), 10_002); // NOPMD - bogus hardcoded IP acceptable in this context + if (socket.getLocalAddress() == null) { + throw new UnknownHostException("bogus exception can be ignored"); + } + ip = socket.getLocalAddress().getHostAddress(); + + if (ip != null) { + return ip; + } + } catch (final SocketException | UnknownHostException e) { + LOGGER.atError().setCause(e).log("getLocalHostName()"); + } + return "localhost"; + } + + private static OpenApiOptions getOpenApiOptions() { + Info applicationInfo = new Info().version("1.0").description(serverName); + return new OpenApiOptions(applicationInfo).path("/swagger-docs").ignorePath("/admin/endpoints", HttpMethod.GET) // Disable documentation + .swagger(new SwaggerOptions("/swagger").title("My Swagger Documentation")) + .reDoc(new ReDocOptions("/redoc").title("My ReDoc Documentation")); + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/admin/RestServerAdmin.java b/server-rest/src/main/java/io/opencmw/server/rest/admin/RestServerAdmin.java new file mode 100644 index 00000000..1ff5a7e5 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/admin/RestServerAdmin.java @@ -0,0 +1,93 @@ +package io.opencmw.server.rest.admin; + +import static io.javalin.apibuilder.ApiBuilder.get; +import static io.javalin.apibuilder.ApiBuilder.post; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javalin.http.Handler; +import io.javalin.plugin.openapi.annotations.OpenApi; +import io.javalin.plugin.openapi.annotations.OpenApiContent; +import io.javalin.plugin.openapi.annotations.OpenApiResponse; +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacRole; +import io.opencmw.server.rest.RestRole; +import io.opencmw.server.rest.RestServer; +import io.opencmw.server.rest.login.LoginController; +import io.opencmw.server.rest.user.RestUserHandler; +import io.opencmw.server.rest.util.MessageBundle; + +/** + * Basic ResetServer admin interface + * @author rstein + */ +@SuppressWarnings("PMD.FieldNamingConventions") +public class RestServerAdmin { // NOPMD - nomen est omen + private static final Logger LOGGER = LoggerFactory.getLogger(RestServerAdmin.class); + private static final String ENDPOINT_ADMIN = "/admin"; + private static final String TEMPLATE_ADMIN = "/velocity/admin/admin.vm"; + + @OpenApi( + description = "endpoint to receive admin requests", + operationId = "serveAdminPage", + summary = "serve ", + tags = { "RestServerAdmin" }, + responses = { + @OpenApiResponse(status = "200", content = @OpenApiContent(type = "text/html")) + }) + private static final Handler serveAdminPage + = ctx -> { + final String userName = LoginController.getSessionCurrentUser(ctx); + final Set roles = LoginController.getSessionCurrentRoles(ctx); + if (!roles.contains(BasicRbacRole.ADMIN)) { + LOGGER.atWarn().addArgument(userName).log("user '{}' does not have the required admin access rights"); + ctx.status(401).result("admin access denied"); + return; + } + RestUserHandler userHandler = RestServer.getUserHandler(); + final Map model = MessageBundle.baseModel(ctx); + model.put("userHandler", userHandler); + model.put("users", userHandler.getAllUserNames()); + model.put("endpoints", RestServer.getEndpoints()); + + ctx.render(TEMPLATE_ADMIN, model); + }; + + @OpenApi( + description = "endpoint to receive admin requests", + operationId = "handleAdminPost", + summary = "POST ", + tags = { "RestServerAdmin" }, + responses = { + @OpenApiResponse(status = "200", content = @OpenApiContent(type = "text/html")) + }) + private static final Handler handleAdminPost + = ctx -> { + final String userName = LoginController.getSessionCurrentUser(ctx); + final Set roles = LoginController.getSessionCurrentRoles(ctx); + if (!roles.contains(BasicRbacRole.ADMIN)) { + LOGGER.atWarn().addArgument(userName).log("user '{}' does not have the required admin access rights"); + ctx.status(401).result("admin access denied"); + return; + } + final Map model = MessageBundle.baseModel(ctx); + + // parse and process admin stuff + ctx.render(TEMPLATE_ADMIN, model); + }; + + /** + * registers the login/logout and locale change listener + */ + public static void register() { + RestServer.getInstance().routes(() -> { + post(ENDPOINT_ADMIN, handleAdminPost, Collections.singleton(new RestRole(BasicRbacRole.ADMIN))); + get(ENDPOINT_ADMIN, serveAdminPage, Collections.singleton(new RestRole(BasicRbacRole.ADMIN))); + }); + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/login/LoginController.java b/server-rest/src/main/java/io/opencmw/server/rest/login/LoginController.java new file mode 100644 index 00000000..6ddd2aab --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/login/LoginController.java @@ -0,0 +1,305 @@ +package io.opencmw.server.rest.login; + +import static io.javalin.apibuilder.ApiBuilder.*; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javalin.core.security.AccessManager; +import io.javalin.http.Context; +import io.javalin.http.Handler; +import io.javalin.plugin.openapi.annotations.OpenApi; +import io.javalin.plugin.openapi.annotations.OpenApiContent; +import io.javalin.plugin.openapi.annotations.OpenApiResponse; +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacRole; +import io.opencmw.server.rest.RestRole; +import io.opencmw.server.rest.RestServer; +import io.opencmw.server.rest.user.RestUserHandler; +import io.opencmw.server.rest.util.MessageBundle; + +@SuppressWarnings("PMD.FieldNamingConventions") +public class LoginController { // NOPMD - nomen est omen + private static final Logger LOGGER = LoggerFactory.getLogger(LoginController.class); + public static final String LOGIN_CONTROLLER = "LoginController"; + private static final String HTTP_200_OK = "200"; + private static final String MIME_HTML = "text/html"; + private static final String MIME_JSON = "text/json"; + private static final String DEFAULT_USER = "anonymous"; + private static final String ENDPOINT_LOGIN = "/login"; + private static final String ENDPOINT_LOGOUT = "/logout"; + private static final String ENDPOINT_CHANGE_PASSWORD = "/changepassword"; + + private static final String ATTR_LOCALE = "locale"; + private static final String ATTR_CURRENT_USER = "currentUser"; + private static final String ATTR_CURRENT_ROLES = "currentRoles"; + private static final String ATTR_LOGIN_REDIRECT = "loginRedirect"; + private static final String ATTR_LOGGED_OUT = "loggedOut"; + + private static final String QUERY_PASSWORD = "password"; + private static final String QUERY_PASSWORD_NEW1 = "passwordNew1"; + private static final String QUERY_PASSWORD_NEW2 = "passwordNew2"; + private static final String QUERY_USERNAME = "username"; + + private static final String AUTHENTICATION_SUCCEEDED = "authenticationSucceeded"; + private static final String AUTHENTICATION_FAILED = "authenticationFailed"; + private static final String AUTHENTICATION_PASSWORD_MISMATCH = "authenticationFailedPasswordsMismatch"; + + private static final String TEMPLATE_LOGIN = "/velocity/login/login.vm"; + private static final String TEMPLATE_PASSWORD_CHANGE = "/velocity/login/changePassword.vm"; + + /** + * Locale change can be initiated from any page The locale is extracted from the + * request and saved to the user's session + */ + private static final Handler handleLocaleChange = ctx -> { + if (ctx.queryParam(ATTR_LOCALE) != null) { + ctx.sessionAttribute(ATTR_LOCALE, ctx.queryParam(ATTR_LOCALE)); + ctx.redirect(ctx.path()); + } + }; + @OpenApi( + description = "endpoint to receive password login request", + operationId = "handleLoginPost", + summary = "POST login command", + tags = { LOGIN_CONTROLLER }, + responses = { + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_HTML)) + , + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_JSON)) + }) + private static final Handler handleLoginPost + = ctx -> { + final Map model = MessageBundle.baseModel(ctx); + final RestUserHandler userHandler = RestServer.getUserHandler(); + + final String userName = ctx.formParam(QUERY_USERNAME); + + if (userHandler.authenticate(userName, ctx.formParam(QUERY_PASSWORD))) { + ctx.sessionAttribute(ATTR_CURRENT_USER, userName); + ctx.sessionAttribute(ATTR_CURRENT_ROLES, userHandler.getUserRolesByUsername(userName)); + + model.put(AUTHENTICATION_SUCCEEDED, true); + model.put(ATTR_CURRENT_USER, userName); + model.put(ATTR_CURRENT_ROLES, userHandler.getUserRolesByUsername(userName)); + + final String loginRedirect = ctx.sessionAttribute(ATTR_LOGIN_REDIRECT); + if (loginRedirect != null) { + ctx.redirect(loginRedirect); + } + } else { + model.put(AUTHENTICATION_FAILED, true); + } + ctx.render(TEMPLATE_LOGIN, model); + }; + + @OpenApi( + description = "endpoint to receive password changes", + operationId = "handleChangePasswordPost", + summary = "POST password change page", + tags = { LOGIN_CONTROLLER }, + responses = { + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_HTML)) + }) + private static final Handler handleChangePasswordPost + = ctx -> { + final Map model = MessageBundle.baseModel(ctx); + + final String userName = ctx.formParam(QUERY_USERNAME); + final String password1 = ctx.formParam(QUERY_PASSWORD_NEW1); + final String password2 = ctx.formParam(QUERY_PASSWORD_NEW2); + if (userName == null || password1 == null || password2 == null) { + model.put(AUTHENTICATION_FAILED, true); + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + return; + } + + if (!checkPasswordCriteria(password1) || !checkPasswordCriteria(password2) || !password1.equals(password2)) { + LOGGER.atWarn().addArgument(userName).log("password do not match for user '{}'"); + model.put(AUTHENTICATION_PASSWORD_MISMATCH, true); + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + return; + } + model.put(AUTHENTICATION_PASSWORD_MISMATCH, false); + + try { + final String password = ctx.formParam(QUERY_PASSWORD); + if (password == null) { + model.put(AUTHENTICATION_FAILED, true); + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + return; + } + + final RestUserHandler userHandler = RestServer.getUserHandler(); + if (userHandler.setPassword(userName, password, password1)) { + ctx.sessionAttribute(ATTR_CURRENT_USER, userName); + ctx.sessionAttribute(ATTR_CURRENT_ROLES, userHandler.getUserRolesByUsername(userName)); + + model.put(AUTHENTICATION_SUCCEEDED, true); + model.put(ATTR_CURRENT_USER, userName); + model.put(ATTR_CURRENT_ROLES, userHandler.getUserRolesByUsername(userName)); + + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + return; + } + + model.put(AUTHENTICATION_FAILED, true); + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + } catch (final SecurityException e) { + LOGGER.atWarn().setCause(e).addArgument(userName).log("may not change password for user '{}'"); + } + model.put(AUTHENTICATION_FAILED, true); + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + }; + + @OpenApi( + description = "endpoint to receive password logout request", + operationId = "handleLogoutPost", + summary = "POST logout command", + tags = { LOGIN_CONTROLLER }, + responses = { + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_HTML)) + , + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_JSON)) + }) + private static final Handler handleLogoutPost + = ctx -> { + ctx.sessionAttribute(ATTR_CURRENT_USER, null); + ctx.sessionAttribute(ATTR_CURRENT_ROLES, null); + ctx.sessionAttribute(ATTR_LOGGED_OUT, "true"); + ctx.redirect(ENDPOINT_LOGIN); + }; + + @OpenApi( + description = "endpoint to serve login page", + operationId = "serveLoginPage", + summary = "GET serve login page (HTML-only)", + tags = { LOGIN_CONTROLLER }, + + // method = HttpMethod.GET, + responses = { + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_HTML)) + }) + private static final Handler serveLoginPage + = ctx -> { + final Map model = MessageBundle.baseModel(ctx); + model.put(ATTR_LOGGED_OUT, removeSessionAttrLoggedOut(ctx)); + ctx.render(TEMPLATE_LOGIN, model); + }; + + @OpenApi( + description = "endpoint to serve password change page", + operationId = "servePasswordChangePage", + summary = "GET serve password change page (HTML-only)", + tags = { LOGIN_CONTROLLER }, + responses = { + @OpenApiResponse(status = HTTP_200_OK, content = @OpenApiContent(type = MIME_HTML)) + }) + private static final Handler servePasswordChangePage + = ctx -> { + final Map model = MessageBundle.baseModel(ctx); + model.put(ATTR_LOGGED_OUT, removeSessionAttrLoggedOut(ctx)); + ctx.render(TEMPLATE_PASSWORD_CHANGE, model); + }; + + /** + * The origin of the request (request.pathInfo()) is saved in the session so the + * user can be redirected back after login + */ + public static final AccessManager accessManager = (handler, ctx, permittedRoles) -> { + final Set userRoles = LoginController.getSessionCurrentRoles(ctx); + final Set permittedRbacRoles = convertRoles(permittedRoles); + final Set intersection = new HashSet<>(permittedRbacRoles); + intersection.retainAll(userRoles); + if (permittedRbacRoles.isEmpty() || permittedRbacRoles.contains(BasicRbacRole.ANYONE) || !intersection.isEmpty()) { + handler.handle(ctx); + } else { + LOGGER.atWarn().addArgument(ctx.path()).addArgument(permittedRbacRoles).addArgument(intersection).log("could not log into '{}' permitted roles {} vs. have {}"); + + // try to login + if (ctx.sessionAttribute(ATTR_CURRENT_USER) == null) { + ctx.sessionAttribute(ATTR_LOGIN_REDIRECT, ctx.path()); + ctx.redirect(ENDPOINT_LOGIN); + } else { + ctx.status(401).result("Unauthorized"); + } + } + }; + + private static Set convertRoles(final Set javalinRoles) { + Set set = new HashSet<>(); + for (final io.javalin.core.security.Role role : javalinRoles) { + if (role instanceof RestRole) { + set.add(((RestRole) role).rbacRole); + } + } + return set; + } + + private LoginController() { + // primarily static helper class + } + + public static Set getSessionCurrentRoles(final Context ctx) { + Object val = ctx.sessionAttribute(ATTR_CURRENT_ROLES); + if (val == null) { + // second attempt mapping to DEFAULT_USER roles + val = RestServer.getUserHandler().getUserRolesByUsername(DEFAULT_USER); + } + if (!(val instanceof Set)) { + return Collections.singleton(BasicRbacRole.NULL); + } + try { + @SuppressWarnings("unchecked") + final Set roles = (Set) val; + return roles; + } catch (final ClassCastException e) { + LOGGER.atError().setCause(e).addArgument(ATTR_CURRENT_ROLES).log("could not cast '{}' attribute to Set -- something fishy is going on"); + } + + return Collections.singleton(BasicRbacRole.NULL); + } + + public static String getSessionCurrentUser(final Context ctx) { + return ctx.sessionAttribute(ATTR_CURRENT_USER); + } + + public static String getSessionLocale(final Context ctx) { + return ctx.sessionAttribute(ATTR_LOCALE); + } + + /** + * registers the login/logout and locale change listener + */ + public static void register() { + RestServer.getInstance().config.accessManager(accessManager); + RestServer.getInstance().routes(() -> { + // before(handleLoginPost) + before(handleLocaleChange); + post(ENDPOINT_LOGIN, handleLoginPost); + post(ENDPOINT_LOGOUT, handleLogoutPost); + post(ENDPOINT_CHANGE_PASSWORD, handleChangePasswordPost); + get(ENDPOINT_LOGIN, serveLoginPage); + get(ENDPOINT_CHANGE_PASSWORD, servePasswordChangePage); + }); + } + + private static boolean checkPasswordCriteria(final String password) { + //TODO: add better password rules + // goal: higher entropy and favour larger number of characters + // rather than complex special characters and/or number combinations + // see security recommendations at: https://xkcd.com/936/ + return password != null && password.length() >= 8; + } + + private static boolean removeSessionAttrLoggedOut(final Context ctx) { + final String loggedOut = ctx.sessionAttribute(ATTR_LOGGED_OUT); + ctx.sessionAttribute(ATTR_LOGGED_OUT, null); + return loggedOut != null; + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/user/RestUser.java b/server-rest/src/main/java/io/opencmw/server/rest/user/RestUser.java new file mode 100644 index 00000000..27a25f20 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/user/RestUser.java @@ -0,0 +1,29 @@ +package io.opencmw.server.rest.user; + +import java.util.Collections; +import java.util.Set; + +import io.opencmw.rbac.RbacRole; + +public class RestUser { + protected final String userName; + protected String salt; + protected String hashedPassword; + private final Set roles; + + public RestUser(final String username, final String salt, final String hashedPassword, final Set roles) { + this.userName = username; + this.salt = salt; + this.hashedPassword = hashedPassword; + this.roles = roles == null ? Collections.emptySet() : Collections.unmodifiableSet(roles); + } + + protected Set getRoles() { + return roles; + } + + @Override + public String toString() { + return "RestUser{" + userName + ", roles=" + roles + "}"; + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/user/RestUserHandler.java b/server-rest/src/main/java/io/opencmw/server/rest/user/RestUserHandler.java new file mode 100644 index 00000000..28a326e6 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/user/RestUserHandler.java @@ -0,0 +1,43 @@ +package io.opencmw.server.rest.user; + +import java.util.Set; + +import io.opencmw.rbac.RbacRole; + +/** + * Basic user handler interface to control access to various routes. + * + * N.B. new implementations may be injected through the RestServer factory. + * + * @author rstein + * @see io.opencmw.server.rest.RestServer#setUserHandler(RestUserHandler) + */ +public interface RestUserHandler { + /** + * Authenticates user against given back-end. + * + * @param username the user name + * @param password the secret password + * @return {@code true} if successful + */ + boolean authenticate(String username, String password); + + Iterable getAllUserNames(); + + RestUser getUserByUsername(String username); + + Set getUserRolesByUsername(String username); + + /** + * Sets new user password. + * + * N.B. Implementation may be implemented or omitted based on the specific back-end. + * + * @param userName existing + * @param oldPassword to verify + * @param newPassword to set + * @throws SecurityException if underlying implementation does not allow to change the password. + * @return {@code true} if successful + */ + boolean setPassword(String userName, String oldPassword, String newPassword) throws SecurityException; //NOPMD - name overload and exception intended +} \ No newline at end of file diff --git a/server-rest/src/main/java/io/opencmw/server/rest/user/RestUserHandlerImpl.java b/server-rest/src/main/java/io/opencmw/server/rest/user/RestUserHandlerImpl.java new file mode 100644 index 00000000..0011e092 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/user/RestUserHandlerImpl.java @@ -0,0 +1,203 @@ +package io.opencmw.server.rest.user; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.jetbrains.annotations.NotNull; +import org.mindrot.jbcrypt.BCrypt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacRole; +import io.opencmw.server.rest.RestServer; + +public class RestUserHandlerImpl implements RestUserHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(RestUserHandlerImpl.class); + private static final String REST_USER_PASSWORD_STORE = "restUserPasswordStore"; + /** + * the password file location is statically allocated so that it cannot (for + * security reasons) be overwritting during run time + */ + private static final String REST_USER_PASSWORD_FILE = getUserPasswordStore(); + + private final Object usersLock = new Object(); + private final RbacRole protoRole; + private List users = Collections.emptyList(); + + public RestUserHandlerImpl(final RbacRole protoRole) { + this.protoRole = protoRole; + } + + /** + * Authenticate the user by hashing the input password using the stored salt, + * then comparing the generated hashed password to the stored hashed password + */ + @Override + public boolean authenticate(@NotNull final String username, @NotNull final String password) { + synchronized (usersLock) { + final RestUser user = getUserByUsername(username); + if (user == null) { + return false; + } + final String hashedPassword = BCrypt.hashpw(password, user.salt); + return hashedPassword.equals(user.hashedPassword); + } + } + + @Override + public Iterable getAllUserNames() { + synchronized (usersLock) { + if (users.isEmpty()) { + readPasswordFile(); + } + return users.stream().map(user -> user.userName).collect(Collectors.toList()); + } + } + + @Override + public RestUser getUserByUsername(final String userName) { + synchronized (usersLock) { + if (users.isEmpty()) { + readPasswordFile(); + } + return users.stream().filter(b -> b.userName.equals(userName)).findFirst().orElse(null); + } + } + + @Override + public Set getUserRolesByUsername(final String userName) { + synchronized (usersLock) { + if (users.isEmpty()) { + readPasswordFile(); + } + RestUser user = getUserByUsername(userName); + if (user != null) { + return user.getRoles(); + } + return Collections.singleton(BasicRbacRole.NULL); + } + } + + public void readPasswordFile() { + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().log("readPasswordFile called"); + } + synchronized (usersLock) { + try (BufferedReader br = REST_USER_PASSWORD_FILE == null ? new BufferedReader(new InputStreamReader(RestServer.class.getResourceAsStream("/DefaultRestUserPasswords.pwd"), StandardCharsets.UTF_8)) // + : Files.newBufferedReader(Paths.get(new File(REST_USER_PASSWORD_FILE).getPath()), StandardCharsets.UTF_8)) { + final List newUserList = new ArrayList<>(10); + String userLine; + int lineCount = 0; + while ((userLine = br.readLine()) != null) { // NOPMD NOSONAR -- early return/continue on purpose + if (userLine.startsWith("#")) { + continue; + } + lineCount++; + parsePasswordLine(newUserList, userLine, lineCount); + } + users = Collections.unmodifiableList(newUserList); + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().log("PasswordFile successfully read"); + } + } catch (IOException e) { + LOGGER.atError().setCause(e).addArgument(REST_USER_PASSWORD_FILE).log("could not read rest user passwords to '{}'"); + } + } + } + + @Override + public boolean setPassword(@NotNull final String userName, @NotNull final String oldPassword, @NotNull final String newPassword) { + if (REST_USER_PASSWORD_FILE == null) { + LOGGER.atWarn().log("cannot set password for default user password store"); + return false; + } + synchronized (usersLock) { + if (authenticate(userName, oldPassword)) { + final RestUser user = getUserByUsername(userName); + if (user == null) { + return false; + } + // N.B. default rounds is 2^10, increase this if necessary to harden passwords + final String newSalt = BCrypt.gensalt(); + final String newHashedPassword = BCrypt.hashpw(newPassword, newSalt); + user.salt = newSalt; + user.hashedPassword = newHashedPassword; + writePasswordFile(); + return true; + } + return false; + } + } + + @SuppressWarnings("unchecked") + public void writePasswordFile() { + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().log("updatePasswordFile called"); + } + if (REST_USER_PASSWORD_FILE == null) { + LOGGER.atWarn().log("cannot write password for default user password store"); + return; + } + synchronized (usersLock) { + final File file = new File(REST_USER_PASSWORD_FILE); + try { + if (file.createNewFile()) { + LOGGER.atInfo().addArgument(REST_USER_PASSWORD_FILE).log("needed to create new password file '{}'"); + } + } catch (SecurityException | IOException e) { + LOGGER.atError().setCause(e).addArgument(REST_USER_PASSWORD_FILE).log("could not create user passwords file '{}'"); + return; + } + + try (BufferedWriter bw = Files.newBufferedWriter(Paths.get(file.getPath()), StandardCharsets.UTF_8)) { + final StringBuilder builder = new StringBuilder(); + for (final RestUser user : users) { + builder.delete(0, builder.length()); // inits and re-uses builder + builder.append(user.userName).append(':').append(user.salt).append(':').append(user.hashedPassword).append(':'); + // write roles + builder.append(protoRole.getRoles(user.getRoles())).append(':'); + bw.write(builder.toString()); + bw.newLine(); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().log("PasswordFile successfully updated"); + } + } catch (IOException e) { + LOGGER.atError().setCause(e).addArgument(REST_USER_PASSWORD_FILE).log("could not store rest user passwords to '{}'"); + } + } + } + + private void parsePasswordLine(@NotNull final List newUserList, final String userLine, final int lineCount) { + try { + final String[] items = userLine.split(":"); + if (items.length < 4) { // NOPMD + LOGGER.atWarn().addArgument(items.length).addArgument(lineCount).addArgument(userLine).log("insufficient arguments ({} < 4)- parsing line {}: '{}'"); + return; + } + newUserList.add(new RestUser(items[0], items[1], items[2], protoRole.getRoles(items[3]))); // NOPMD - needed + } catch (Exception e) { // NOPMD - catch generic exception since a faulty login should not crash the rest of the REST service + LOGGER.atWarn().setCause(e).addArgument(lineCount).addArgument(userLine).log("could not parse line {}: '{}'"); + } + } + + private static String getUserPasswordStore() { + final String passWordStore = System.getProperty(REST_USER_PASSWORD_STORE); + if (passWordStore == null) { + LOGGER.atWarn().log("using internal UserPasswordStore -- PLEASE CHANGE FOR PRODUCTION -- THIS IS UNSAFE PRACTICE"); + } + return passWordStore; + } +} diff --git a/server-rest/src/main/java/io/opencmw/server/rest/util/CombinedHandler.java b/server-rest/src/main/java/io/opencmw/server/rest/util/CombinedHandler.java new file mode 100644 index 00000000..6cf1b4d6 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/util/CombinedHandler.java @@ -0,0 +1,111 @@ +package io.opencmw.server.rest.util; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import javax.servlet.AsyncEvent; +import javax.servlet.AsyncListener; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javalin.core.util.Header; +import io.javalin.http.Context; +import io.javalin.http.Handler; +import io.javalin.http.sse.SseClient; +import io.opencmw.MimeType; +import io.opencmw.server.rest.RestServer; + +/** + * Combined GET and SSE request handler. + * + * N.B. This based on an original idea/implementation found in Javalin's {@link io.javalin.http.sse.SseHandler}. + * + * @author rstein + * + * @see io.javalin.http.sse.SseHandler + */ +public class CombinedHandler implements Handler { + private static final Logger LOGGER = LoggerFactory.getLogger(CombinedHandler.class); + private final Handler getHandler; + private BiConsumer sseClientConnectHandler; + + private final Consumer clientConsumer = client -> { + // TODO: upgrade to path & query matching - for the time being only path @see also MajordomoRestPlugin + // final String queryString = client.ctx.queryString() == null ? "" : ("?" + client.ctx.queryString()) + // final String endPointName = StringUtils.stripEnd(client.ctx.path(), "/") + queryString + final String endPointName = StringUtils.stripEnd(client.ctx.path(), "/"); + + RestServer.getEventClients(endPointName).add(client); + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(client.ctx.req.getRemoteHost()).addArgument(endPointName).log("added SSE client: '{}' to route '{}'"); + } + + if (sseClientConnectHandler != null) { + sseClientConnectHandler.accept(client, SseState.CONNECTED); + } + client.sendEvent("connected", "Hello, new SSE client " + client.ctx.req.getRemoteHost()); + + client.onClose(() -> { + if (sseClientConnectHandler != null) { + sseClientConnectHandler.accept(client, SseState.DISCONNECTED); + } + RestServer.getEventClients(endPointName).remove(client); + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(client.ctx.req.getRemoteHost()).addArgument(endPointName).log("removed client: '{}' from route '{}'"); + } + }); + }; + + public CombinedHandler(@NotNull Handler getHandler) { + this(getHandler, null); + } + + public CombinedHandler(@NotNull Handler getHandler, BiConsumer sseClientConnectHandler) { + this.getHandler = getHandler; + this.sseClientConnectHandler = sseClientConnectHandler; + } + + @Override + public void handle(@NotNull Context ctx) throws Exception { + if (MimeType.EVENT_STREAM.equals(RestServer.getRequestedMimeProtocol(ctx))) { + ctx.res.setStatus(200); + ctx.res.setCharacterEncoding("UTF-8"); + ctx.res.setContentType(MimeType.EVENT_STREAM.toString()); + ctx.res.addHeader(Header.CONNECTION, "close"); + ctx.res.addHeader(Header.CACHE_CONTROL, "no-cache"); + ctx.res.flushBuffer(); + + ctx.req.startAsync(ctx.req, ctx.res); + ctx.req.getAsyncContext().setTimeout(0); + clientConsumer.accept(new SseClient(ctx)); + + ctx.req.getAsyncContext().addListener(new AsyncListener() { + @Override + public void onComplete(AsyncEvent event) { /* not needed */ + } + @Override + public void onError(AsyncEvent event) { + event.getAsyncContext().complete(); + } + @Override + public void onStartAsync(AsyncEvent event) { /* not needed */ + } + @Override + public void onTimeout(AsyncEvent event) { + event.getAsyncContext().complete(); + } + }); + return; + } + + getHandler.handle(ctx); + } + + public enum SseState { + CONNECTED, + DISCONNECTED + } +} \ No newline at end of file diff --git a/server-rest/src/main/java/io/opencmw/server/rest/util/MessageBundle.java b/server-rest/src/main/java/io/opencmw/server/rest/util/MessageBundle.java new file mode 100644 index 00000000..851d7956 --- /dev/null +++ b/server-rest/src/main/java/io/opencmw/server/rest/util/MessageBundle.java @@ -0,0 +1,38 @@ +package io.opencmw.server.rest.util; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.ResourceBundle; + +import io.javalin.http.Context; +import io.opencmw.server.rest.RestServer; + +public class MessageBundle { + private static final String ATTR_CURRENT_MESSAGES = "msg"; + private static final String ATTR_CURRENT_USER = "currentUser"; + private static final String ATTR_CURRENT_ROLES = "currentRoles"; + private final ResourceBundle messages; + + public MessageBundle(final String languageTag) { + final Locale locale = languageTag == null ? Locale.ENGLISH : new Locale(languageTag); + messages = ResourceBundle.getBundle("localisation/messages", locale); + } + + public String get(final String message) { + return messages.getString(message); + } + + public final String get(final String key, final Object... args) { + return MessageFormat.format(get(key), args); + } + + public static Map baseModel(Context ctx) { + final Map model = new HashMap<>(); // NOPMD - thread-safe usage + model.put(ATTR_CURRENT_MESSAGES, new MessageBundle(RestServer.getSessionLocale(ctx))); + model.put(ATTR_CURRENT_USER, RestServer.getSessionCurrentUser(ctx)); + model.put(ATTR_CURRENT_ROLES, RestServer.getSessionCurrentRoles(ctx)); + return model; + } +} diff --git a/server-rest/src/main/resources/DefaultRestUserPasswords.pwd b/server-rest/src/main/resources/DefaultRestUserPasswords.pwd new file mode 100644 index 00000000..ad7374ea --- /dev/null +++ b/server-rest/src/main/resources/DefaultRestUserPasswords.pwd @@ -0,0 +1,6 @@ +# demo password file -- please change for production since these 'password's are not safe!! +admin:$2a$10$h.dl5J86rGH7I8bD9bZeZe:$2a$10$h.dl5J86rGH7I8bD9bZeZeci0pDt0.VwFTGujlnEaZXPf/q7vM5wO:ADMIN: +anonymous:$2a$10$e0MYzXyjpJS7Pd0RVvHwHe:$2a$10$e0MYzXyjpJS7Pd0RVvHwHe1HlCS4bZJ18JuywdEMLT83E1KDmUhCy:READ_WRITE: +alex:$2a$10$rQu1pxN6wCsS.bf6zYObX.:$2a$10$rQu1pxN6wCsS.bf6zYObX.QdkJN85dFpkcrzy1xwWK57ToqpKEVBy:READ_WRITE: +bernd:$2a$10$j2l8uniY45FyKvZLy3ZvaO:$2a$10$j2l8uniY45FyKvZLy3ZvaORFZjPHuGr69Rb/rRCs0W2aPdIfH99pi:READ_ONLY: +zoe:$2a$10$vLiwSkM2/krIqO1eDhGgcu:$2a$10$vLiwSkM2/krIqO1eDhGgcuRv/vJxE8Sh4xePZkBKKZeswP2pTzlAi:ANYONE: diff --git a/server-rest/src/main/resources/keystore.jks b/server-rest/src/main/resources/keystore.jks new file mode 100644 index 00000000..9cf0969c Binary files /dev/null and b/server-rest/src/main/resources/keystore.jks differ diff --git a/server-rest/src/main/resources/keystore.pwd b/server-rest/src/main/resources/keystore.pwd new file mode 100644 index 00000000..7c3a4983 --- /dev/null +++ b/server-rest/src/main/resources/keystore.pwd @@ -0,0 +1 @@ +nopassword diff --git a/server-rest/src/main/resources/localisation/messages_de.properties b/server-rest/src/main/resources/localisation/messages_de.properties new file mode 100644 index 00000000..d5090f9f --- /dev/null +++ b/server-rest/src/main/resources/localisation/messages_de.properties @@ -0,0 +1,36 @@ +## Common +LOCALE=de +COMMON_TITLE=Ze OpenCMW Serivce +COMMON_FOOTER_TEXT=Dieser Service verwendet ZeroMQ, Javalin und OpenCMW
Bitte unterstützen Sie Freie Software und die Public Money? Public Code! Kampagne. +COMMON_NAV_ALLIMAGES=Clipboard Galerie +COMMON_NAV_UPLOAD=Hinaufladen +COMMON_NAV_ADMIN=Chef +COMMON_NAV_LOGIN=Innlogg +COMMON_NAV_LOGOUT=Auslogg +ERROR_400_BAD_REQUEST=Schlechte GET oder SET Anfrage(error 400) +ERROR_401_UNAUTHORISED=Verboten! (error 401) +ERROR_403_ACCESS_DENIED=!!Zutritt Verboten!! (error 403) +ERROR_404_NOT_FOUND=Ve cannot find ze page you are looking for (error 404) +## Login +LOGIN_HEADING=Innlogg +LOGIN_INSTRUCTIONS=Please enter Dein username und Parole. +LOGIN_AUTH_SUCCEEDED=You''re logged in as ''{0}''. +LOGIN_AUTH_FAILED=Ze login informazion you zuplied vas incorrect. +LOGIN_AUTH_FAILED_PASSWORD_MISMATCH=Ze login informazion you zuplied dit not match. +LOGIN_LOGGED_OUT=You have been logged aus. +LOGIN_LABEL_USERNAME=Username +LOGIN_LABEL_PASSWORD=Parole +LOGIN_BUTTON_LOGIN=Innlogg +LOGIN_HEADING_CHANGE_PASSWORD=Parolen Änderung +LOGIN_LABEL_PASSWORD_OLD=olle Parole +LOGIN_LABEL_PASSWORD_NEW1=nieuve Parole +LOGIN_LABEL_PASSWORD_NEW2=nieuve Parole (wirklich) +LOGIN_BUTTON_CHANGE_PASSWORD=jetzt los +## Images +CLIPBOARD_HEADING_ALL=Alle Bildchen +CLIPBOARD_DATA = Keine Bildchen +CLIPBOARD_SUB_CATEGORIES = Unterrubriken +CLIPBOARD_CAPTION={0}
{1} +CLIPBOARD_UPLOAD_HEADING=Upload Data +CLIPBOARD_UPLOAD_INSTRUCTIONS=Please provide a export topic name and data to be exported: +CLIPBOARD_UPLOAD_BUTTON=submit \ No newline at end of file diff --git a/server-rest/src/main/resources/localisation/messages_en.properties b/server-rest/src/main/resources/localisation/messages_en.properties new file mode 100644 index 00000000..2a97de75 --- /dev/null +++ b/server-rest/src/main/resources/localisation/messages_en.properties @@ -0,0 +1,36 @@ +## Common +LOCALE=en +COMMON_TITLE=OpenCMW Service +COMMON_FOOTER_TEXT=This service uses ZeroMQ, Javalin and OpenCMW
Please support Free Software and the Public Money? Public Code! campaign. +COMMON_NAV_ALLIMAGES=Clipboard Gallery +COMMON_NAV_UPLOAD=Upload +COMMON_NAV_ADMIN=Admin +COMMON_NAV_LOGIN=Log in +COMMON_NAV_LOGOUT=Log out +ERROR_400_BAD_REQUEST=bad GET or SET request(error 400) +ERROR_401_UNAUTHORISED=unauthorised access (error 401) +ERROR_403_ACCESS_DENIED=access denied (error 403) +ERROR_404_NOT_FOUND=We cannot find the page you're looking for (error 404) +## Login +LOGIN_HEADING=Login +LOGIN_INSTRUCTIONS=Please enter your username and password. +LOGIN_AUTH_SUCCEEDED=You''re logged in as ''{0}''. +LOGIN_AUTH_FAILED=The login information you supplied was incorrect. +LOGIN_AUTH_FAILED_PASSWORD_MISMATCH=The provided passwords do not match +LOGIN_LOGGED_OUT=You have been logged out. +LOGIN_LABEL_USERNAME=username +LOGIN_LABEL_PASSWORD=password +LOGIN_BUTTON_LOGIN=Log in +LOGIN_HEADING_CHANGE_PASSWORD=Change Password +LOGIN_LABEL_PASSWORD_OLD=old password +LOGIN_LABEL_PASSWORD_NEW1=new password +LOGIN_LABEL_PASSWORD_NEW2=new password (verify) +LOGIN_BUTTON_CHANGE_PASSWORD=change password +## Images +CLIPBOARD_HEADING_ALL=All Snapshots +CLIPBOARD_DATA = Non-Image Data +CLIPBOARD_SUB_CATEGORIES = sub-categories +CLIPBOARD_CAPTION={0}
{1} +CLIPBOARD_UPLOAD_HEADING=Upload Data +CLIPBOARD_UPLOAD_INSTRUCTIONS=Please provide a export topic name and data to be exported: +CLIPBOARD_UPLOAD_BUTTON=submit diff --git a/server-rest/src/main/resources/public/img/english.png b/server-rest/src/main/resources/public/img/english.png new file mode 100644 index 00000000..3a170eec Binary files /dev/null and b/server-rest/src/main/resources/public/img/english.png differ diff --git a/server-rest/src/main/resources/public/img/german.png b/server-rest/src/main/resources/public/img/german.png new file mode 100644 index 00000000..fd64cf59 Binary files /dev/null and b/server-rest/src/main/resources/public/img/german.png differ diff --git a/server-rest/src/main/resources/public/img/logo_b.png b/server-rest/src/main/resources/public/img/logo_b.png new file mode 100644 index 00000000..c3d88782 Binary files /dev/null and b/server-rest/src/main/resources/public/img/logo_b.png differ diff --git a/server-rest/src/main/resources/public/img/logo_w.png b/server-rest/src/main/resources/public/img/logo_w.png new file mode 100644 index 00000000..4c24d9de Binary files /dev/null and b/server-rest/src/main/resources/public/img/logo_w.png differ diff --git a/server-rest/src/main/resources/public/main.css b/server-rest/src/main/resources/public/main.css new file mode 100644 index 00000000..44377b66 --- /dev/null +++ b/server-rest/src/main/resources/public/main.css @@ -0,0 +1,397 @@ +* { + box-sizing: border-box; +} + +html { + margin: 0; + padding: 0; + font-family: Tahoma, Arial, sans-serif; + position: relative; + min-height: 100%; +} + +body { + margin: 0; + font-family: Tahoma, Arial, sans-serif; + padding: 0 0 40px 0; + color: #333; + background: #f9f9f9; +} + +.embeddedFrame { + width: 100%; + min-height: 110vh; +} + +h1, +h2, +h3, +h4 { + font-family: monospace; + font-weight: 300; + color: #444; +} + +small { + color: #555; +} + +header { + background: #00599c; + border-bottom: 5px solid #ffb446; + box-shadow: 0 1px 0 0 rgba(0, 0, 0, 0.6); +} + +nav { + padding: 15px; + margin: 0 auto; + max-width: 1200px; + position: relative; +} + +nav #menu { + font-family: Comic Sans MS, Tahoma, Arial, sans-serif; + margin-top: 20px; + float: right; +} + +a { + text-decoration: none; + color: #008ab4; +} + +#menu li { + float: left; + margin: 0 10px; +} + +#menu li a, +#logout { + background: transparent; + cursor: pointer; + border: 0; + font-size: 16px; + display: inline-block; + color: #fff; + font-family: Comic Sans MS, Tahoma, Arial, sans-serif; + text-align: center; + height: 30px; + line-height: 30px; + padding: 0 10px; + text-decoration: none; +} + +#logo { + max-height: 50px; +} + +#chooseLanguage { + top: 15px; + right: 35px; + position: absolute; +} + +#chooseLanguage li { + float: left; +} + +#chooseLanguage button { + cursor: pointer; + margin-left: 8px; + width: 18px; + height: 18px; + border-radius: 9px; + opacity: 0.6; + border: 1px solid #222; + background-size: 100%; +} + +#chooseLanguage button:hover { + opacity: 0.8; +} + +#main { + max-width: 1200px; + margin: 0 auto; + padding: 25px; +} + +#content { + padding: 15px 20px; + background: #fff; + box-shadow: 0 1px 5px rgba(0, 0, 0, 0.25); +} + +#footer { + position: absolute; + left: 0; + bottom: 0; + height: 50px; + line-height: 18px; + width: 100%; + text-align: center; + background: #fff; + border-top: 2px solid #ffb446; + font-size: 14px; +} + +nav li, +nav ul { + margin: 0; + padding: 0; + list-style-type: none; +} + +/* Needlessly fancy menu hover effect */ +#menu li { + position: relative; +} + +#menu li a::after, #logout:after { + position: absolute; + top: 28px; + left: 0; + width: 100%; + height: 4px; + background: rgba(255, 255, 255, 0.5); + border-radius: 5px; + content: ''; + opacity: 0; + -webkit-transition: opacity 0.3s, -webkit-transform 0.3s; + transition: opacity 0.3s, transform 0.3s; + -webkit-transform: translateY(10px); + transform: translateY(10px); +} + +#logout:hover::after, #logout:focus::after, #menu li a:hover::after, #menu li a:focus::after { + opacity: 1; + -webkit-transform: translateY(0); + transform: translateY(0); +} + +/* Very basic grid */ +.row { + width: 100%; + overflow: auto; +} + +.row > * { + float: left; +} + +.row-2 .col { + width: 49%; +} + +.row-2 .col:nth-child(odd) { + margin: 0% 1% 0% 0%; +} + +.row-2 .col:nth-child(even) { + margin: 0% 0% 0% 1%; +} + +.row-3 .col { + width: 32%; +} + +.row-3 .col:nth-child(3n+1) { + margin: 0% 1% 0% 0%; +} + +.row-3 .col:nth-child(3n+2) { + margin: 0% 1% 0% 1%; +} + +.row-3 .col:nth-child(3n+3) { + margin: 0% 0% 0% 1%; +} + +@media screen and (max-width: 550px) { + .row .col:nth-child(n) { + width: 100%; + margin-right: 0; + margin-left: 0; + } +} + +.col img { + display: block; + width: 100%; +} + +/* image related stuff */ +a.image { + display: block; + text-align: center; + text-decoration: none; + color: #333; + padding-top: 10px; + padding-left: 10px; + padding-right: 10px; + border-radius: 5px; +} + +a.image:hover { + background: #e2e9f5; +} + +.imageCover { + padding: 10px; + display: flex; + align-items: center; + justify-content: center; +} + +.imageCover img { + border-radius: 5px; + min-height: 200px; + max-height: 200px; + height: auto; + width: auto; + object-fit: contain; +} + +.imageCover img { + margin-top: 20px; + border-radius: 10px; + width: 100%; +} + +a.link { + text-decoration: none; + color: #008ab4; + padding: 0; + border-radius: 5px; +} + +a.link:hover { + background: #e2e9f5; +} + +/* Login Form */ +#loginForm { + max-width: 400px; + margin: 0 auto; +} + +#loginForm label { + display: block; + width: 100%; +} + +#loginForm input { + border: 1px solid #ddd; + padding: 8px 12px; + width: 100%; + border-radius: 3px; + margin: 2px 0 20px 0; +} + +#loginForm input[type="submit"] { + color: white; + background: #00599c; + border: 0; + cursor: pointer; +} + +.notification { + padding: 10px; + background: #333; + color: white; + border-radius: 3px; +} + +.good.notification { + background: #008900; +} + +.bad.notification { + background: #bb00; +} + +.skip-link { + position: absolute; + top: -40px; + left: 0; + background: #0000; + color: white; + padding: 8px; + z-index: 100; +} + +.skip-link:focus { + top: 0; +} + +/* Login Form */ +#propertyView { + max-width: 400px; + margin: 0 auto; +} + +#defaultProperty { + font-family: sans-serif; +} + +#defaultProperty fieldset { + border-style:solid; + border-radius: 5px; + border-color:#F5F5F5; + padding-top: 2px; + padding-bottom: 2px; + padding-left: 5px; + /*overflow: hidden;*/ +} + +#defaultProperty legend { + background: #00599c; + color: #fff; + padding: 2px 10px ; + font-size: 14px; + border-radius: 5px; + box-shadow: 0 0 0 2px #ddd; + margin-left: 5px; +} + +#defaultProperty label { + display: inline; + width: 20%; + min-width: 20%; +} + +#defaultProperty label[id="unit"] { + display: inline; + width: 10%; + min-width: 10%; +} + +#defaultProperty label[id="description"] { + display: inline; + width: 20%; + min-width: 20%; +} + +#defaultProperty input { + border: 1px solid #ddd; + padding: 8px 12px; + width: 50%; + border-radius: 3px; + margin: 2px 0 20px 0; +} + +#defaultProperty input[type="submit"] { + color: white; + width: 5%; + background: #00599c; + border: 0; + cursor: pointer; +} + +#defaultProperty input[value="SUBSCRIBE"] { + color: white; + width: 10%; + background: #00599c; + border: 0; + cursor: pointer; +} diff --git a/server-rest/src/main/resources/velocity/admin/admin.vm b/server-rest/src/main/resources/velocity/admin/admin.vm new file mode 100644 index 00000000..f826f882 --- /dev/null +++ b/server-rest/src/main/resources/velocity/admin/admin.vm @@ -0,0 +1,14 @@ +#parse("/velocity/layout.vm") +#@mainLayout() + +

Admin Interface

+Users: +
    +#foreach($user in $users) +
  • $user with roles $userHandler.getUserRolesByUsername($user)
    • +#end +
+
+Direct access: /admin/endpoints + +#end \ No newline at end of file diff --git a/server-rest/src/main/resources/velocity/clipboard/all.vm b/server-rest/src/main/resources/velocity/clipboard/all.vm new file mode 100644 index 00000000..d825a8ff --- /dev/null +++ b/server-rest/src/main/resources/velocity/clipboard/all.vm @@ -0,0 +1,25 @@ +#parse("/velocity/layout.vm") +#@ mainLayout()

$msg.get("CLIPBOARD_HEADING_ALL") #if (!$category.isEmpty()) - $category #end

+#if (!$categories.isEmpty()) + $msg.get("CLIPBOARD_SUB_CATEGORIES"): +#foreach ($category in $categories) + $category +#end + +#end +
+#foreach ($image in $images) +
+#set($imgStr = "/clipboard/" + $category + $image.getExportNameData()) + $image.getExportNameData()
$msg.get("CLIPBOARD_CAPTION", $image.getExportName(), $image.getTimeStampCreationString())
( long - polling, SSE)
+#end +
+#if (!$data.isEmpty()) +

$msg.get("CLIPBOARD_DATA")

+#foreach ($dataItem in $data) +#set($dataStr = "/clipboard/$category" + "$dataItem.getExportNameData()") + $dataItem.getExportNameData() +#end +
+#end +#end diff --git a/server-rest/src/main/resources/velocity/clipboard/one_long.vm b/server-rest/src/main/resources/velocity/clipboard/one_long.vm new file mode 100644 index 00000000..741a4cdd --- /dev/null +++ b/server-rest/src/main/resources/velocity/clipboard/one_long.vm @@ -0,0 +1,86 @@ + + + + $title + + + + + + + + + + + + + + + + $imageSource + + + + \ No newline at end of file diff --git a/server-rest/src/main/resources/velocity/clipboard/one_sse.vm b/server-rest/src/main/resources/velocity/clipboard/one_sse.vm new file mode 100644 index 00000000..93ccf2cb --- /dev/null +++ b/server-rest/src/main/resources/velocity/clipboard/one_sse.vm @@ -0,0 +1,111 @@ + + + + $title + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server-rest/src/main/resources/velocity/clipboard/upload.vm b/server-rest/src/main/resources/velocity/clipboard/upload.vm new file mode 100644 index 00000000..19f2cd35 --- /dev/null +++ b/server-rest/src/main/resources/velocity/clipboard/upload.vm @@ -0,0 +1,17 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +
+ #if($authenticationFailed) +

$msg.get("LOGIN_AUTH_FAILED")

+ #elseif($authenticationSucceeded) +

$msg.get("LOGIN_AUTH_SUCCEEDED", $currentUser)

+ #elseif($loggedOut) +

$msg.get("LOGIN_LOGGED_OUT")

+ #end +

$msg.get("CLIPBOARD_UPLOAD_HEADING")

+

$msg.get("CLIPBOARD_UPLOAD_INSTRUCTIONS", "/index")

+ + + +
+#end diff --git a/server-rest/src/main/resources/velocity/errors/accessDenied.vm b/server-rest/src/main/resources/velocity/errors/accessDenied.vm new file mode 100644 index 00000000..fcf59529 --- /dev/null +++ b/server-rest/src/main/resources/velocity/errors/accessDenied.vm @@ -0,0 +1,4 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +

$msg.get("ERROR_403_ACCESS_DENIED")

+#end diff --git a/server-rest/src/main/resources/velocity/errors/badRequest.vm b/server-rest/src/main/resources/velocity/errors/badRequest.vm new file mode 100644 index 00000000..9ef54e65 --- /dev/null +++ b/server-rest/src/main/resources/velocity/errors/badRequest.vm @@ -0,0 +1,8 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +

$msg.get("ERROR_400_BAD_REQUEST")

+#if ($exceptionText) +Service '$service' error reply:
+ $exceptionText +#end +#end \ No newline at end of file diff --git a/server-rest/src/main/resources/velocity/errors/notFound.vm b/server-rest/src/main/resources/velocity/errors/notFound.vm new file mode 100644 index 00000000..15e3e424 --- /dev/null +++ b/server-rest/src/main/resources/velocity/errors/notFound.vm @@ -0,0 +1,4 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +

$msg.get("ERROR_404_NOT_FOUND")

+#end diff --git a/server-rest/src/main/resources/velocity/errors/unauthorised.vm b/server-rest/src/main/resources/velocity/errors/unauthorised.vm new file mode 100644 index 00000000..9ad16981 --- /dev/null +++ b/server-rest/src/main/resources/velocity/errors/unauthorised.vm @@ -0,0 +1,4 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +

$msg.get("ERROR_401_UNAUTHORISED")

+#end \ No newline at end of file diff --git a/server-rest/src/main/resources/velocity/layout.vm b/server-rest/src/main/resources/velocity/layout.vm new file mode 100644 index 00000000..6dc11fe0 --- /dev/null +++ b/server-rest/src/main/resources/velocity/layout.vm @@ -0,0 +1,56 @@ +#macro(mainLayout) + + + + $msg.get("COMMON_TITLE") + + + + + + + +
+ +
+
+
+ $bodyContent +
+
+
+ $msg.get("COMMON_FOOTER_TEXT") +
+ + +#end diff --git a/server-rest/src/main/resources/velocity/layoutNoFrame.vm b/server-rest/src/main/resources/velocity/layoutNoFrame.vm new file mode 100644 index 00000000..52f04a56 --- /dev/null +++ b/server-rest/src/main/resources/velocity/layoutNoFrame.vm @@ -0,0 +1,18 @@ +#macro(mainLayoutNoFrame) + + + + $msg.get("COMMON_TITLE") + + + + + + + +
+ $bodyContent +
+ + +#end diff --git a/server-rest/src/main/resources/velocity/login/changePassword.vm b/server-rest/src/main/resources/velocity/login/changePassword.vm new file mode 100644 index 00000000..e7baa511 --- /dev/null +++ b/server-rest/src/main/resources/velocity/login/changePassword.vm @@ -0,0 +1,28 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +
+ #if($authenticationFailedPasswordsMismatch) +

$msg.get("LOGIN_AUTH_FAILED_PASSWORD_MISMATCH")

+ #end + #if($authenticationFailed) +

$msg.get("LOGIN_AUTH_FAILED")

+ #elseif($authenticationSucceeded) +

$msg.get("LOGIN_AUTH_SUCCEEDED", $currentUser)

+ #elseif($loggedOut) +

$msg.get("LOGIN_LOGGED_OUT")

+ #end +

$msg.get("LOGIN_HEADING_CHANGE_PASSWORD")

+

$msg.get("LOGIN_INSTRUCTIONS", "/index")

+ + + + + + + + #if($loginRedirect) + + #end + +
+#end diff --git a/server-rest/src/main/resources/velocity/login/login.vm b/server-rest/src/main/resources/velocity/login/login.vm new file mode 100644 index 00000000..689c5962 --- /dev/null +++ b/server-rest/src/main/resources/velocity/login/login.vm @@ -0,0 +1,22 @@ +#parse("/velocity/layout.vm") +#@mainLayout() +
+ #if($authenticationFailed) +

$msg.get("LOGIN_AUTH_FAILED")

+ #elseif($authenticationSucceeded) +

$msg.get("LOGIN_AUTH_SUCCEEDED", $currentUser)

+ #elseif($loggedOut) +

$msg.get("LOGIN_LOGGED_OUT")

+ #end +

$msg.get("LOGIN_HEADING")

+

$msg.get("LOGIN_INSTRUCTIONS", "/")
($msg.get("LOGIN_BUTTON_CHANGE_PASSWORD"))

+ + + + + #if($loginRedirect) + + #end + +
+#end diff --git a/server-rest/src/main/resources/velocity/property/defaultTextPropertyLayout.vm b/server-rest/src/main/resources/velocity/property/defaultTextPropertyLayout.vm new file mode 100644 index 00000000..91714521 --- /dev/null +++ b/server-rest/src/main/resources/velocity/property/defaultTextPropertyLayout.vm @@ -0,0 +1,11 @@ +#if (!$noMenu) + #parse("/velocity/layout.vm") + #@mainLayout() + $textBody + #end +#else + #parse("/velocity/layoutNoFrame.vm") + #@mainLayoutNoFrame() + $textBody + #end +#end \ No newline at end of file diff --git a/server-rest/src/main/resources/velocityconfig/velocity_implicit.vm b/server-rest/src/main/resources/velocityconfig/velocity_implicit.vm new file mode 100644 index 00000000..f854801a --- /dev/null +++ b/server-rest/src/main/resources/velocityconfig/velocity_implicit.vm @@ -0,0 +1,9 @@ +#* @implicitly included * # +#* @vtlvariable name = "msg" type = "io.opencmw.server.rest.util.MessageBundle" * # +#* @vtlvariable name = "users" type = "java.lang.Iterable" * # +#* @vtlvariable name = "currentUser" type = "java.lang.String" * # +#* @vtlvariable name = "currentRoles" type = "java.lang.List" * # +#* @vtlvariable name = "loggedOut" type = "java.lang.String" * # +#* @vtlvariable name = "authenticationFailed" type = "java.lang.String" * # +#* @vtlvariable name = "authenticationSucceeded" type = "java.lang.String" * # +#* @vtlvariable name = "loginRedirect" type = "java.lang.String" * # diff --git a/server-rest/src/test/java/io/opencmw/server/rest/MajordomoRestPluginTests.java b/server-rest/src/test/java/io/opencmw/server/rest/MajordomoRestPluginTests.java new file mode 100644 index 00000000..c9ae3e57 --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/MajordomoRestPluginTests.java @@ -0,0 +1,228 @@ +package io.opencmw.server.rest; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.net.ssl.*; + +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.MimeType; +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.server.MajordomoBroker; +import io.opencmw.server.rest.test.HelloWorldService; +import io.opencmw.server.rest.test.ImageService; + +import okhttp3.Headers; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.sse.EventSource; +import okhttp3.sse.EventSourceListener; +import okhttp3.sse.EventSources; +import zmq.util.Utils; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MajordomoRestPluginTests { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoRestPluginTests.class); + private MajordomoBroker primaryBroker; + private String brokerRouterAddress; + private MajordomoBroker secondaryBroker; + private String secondaryBrokerRouterAddress; + private OkHttpClient okHttp; + + @BeforeAll + void init() throws IOException { + okHttp = getUnsafeOkHttpClient(); // N.B. ignore SSL certificates + primaryBroker = new MajordomoBroker("PrimaryBroker", "", BasicRbacRole.values()); + brokerRouterAddress = primaryBroker.bind("mdp://localhost:" + Utils.findOpenPort()); + primaryBroker.bind("mds://localhost:" + Utils.findOpenPort()); + MajordomoRestPlugin restPlugin = new MajordomoRestPlugin(primaryBroker.getContext(), "My test REST server", "*:8080", BasicRbacRole.ADMIN); + primaryBroker.start(); + restPlugin.start(); + LOGGER.atInfo().log("Broker and REST plugin started"); + + // start simple test services/properties + final HelloWorldService helloWorldService = new HelloWorldService(primaryBroker.getContext()); + helloWorldService.start(); + final ImageService imageService = new ImageService(primaryBroker.getContext(), 100); + imageService.start(); + + // TODO: add OpenCMW client requesting binary and json models + + // second broker to test DNS functionalities + secondaryBroker = new MajordomoBroker("SecondaryTestBroker", brokerRouterAddress, BasicRbacRole.values()); + secondaryBrokerRouterAddress = secondaryBroker.bind("tcp://*:" + Utils.findOpenPort()); + secondaryBroker.start(); + } + + @AfterAll + void finish() { + secondaryBroker.stopBroker(); + primaryBroker.stopBroker(); + } + + @ParameterizedTest + @ValueSource(strings = { "http://localhost:8080", "https://localhost:8443" }) + void testDns(final String address) throws IOException { + final Request request = new Request.Builder().url(address + "/mmi.dns?noMenu").addHeader("accept", MimeType.HTML.getMediaType()).get().build(); + final Response response = okHttp.newCall(request).execute(); + final String body = Objects.requireNonNull(response.body()).string(); + + assertThat(body, containsString(brokerRouterAddress)); + assertThat(body, containsString(secondaryBrokerRouterAddress)); + assertThat(body, containsString("http://localhost:8080")); + } + + @ParameterizedTest + @EnumSource(value = MimeType.class, names = { "HTML", "BINARY", "JSON", "CMWLIGHT", "TEXT", "UNKNOWN" }) + void testGet(final MimeType contentType) throws IOException { + final Request request = new Request.Builder().url("http://localhost:8080/helloWorld?noMenu").addHeader("accept", contentType.getMediaType()).get().build(); + final Response response = okHttp.newCall(request).execute(); + final Headers header = response.headers(); + final String body = Objects.requireNonNull(response.body()).string(); + + if (contentType == MimeType.TEXT) { + assertEquals(MimeType.HTML.getMediaType(), header.get("Content-Type"), "you get the content type you asked for"); + } else { + assertEquals(contentType.getMediaType(), header.get("Content-Type"), "you get the content type you asked for"); + } + assertThat(body, containsString("byteReturnType")); + assertThat(body, containsString("Hello World! The local time is:")); + + switch (contentType) { + case JSON: + assertThat(body, containsString("\"byteReturnType\": 42,")); + break; + case TEXT: + default: + break; + } + } + + @ParameterizedTest + @EnumSource(value = MimeType.class, names = { "HTML", "BINARY", "JSON", "CMWLIGHT", "TEXT", "UNKNOWN" }) + void testGetException(final MimeType contentType) throws IOException { + final Request request = new Request.Builder().url("http://localhost:8080/mmi.openapi?noMenu").addHeader("accept", contentType.getMediaType()).get().build(); + final Response response = okHttp.newCall(request).execute(); + final Headers header = response.headers(); + final String body = Objects.requireNonNull(response.body()).string(); + switch (contentType) { + case HTML: + case TEXT: + assertEquals(200, response.code()); + assertThat(body, containsString("java.util.concurrent.ExecutionException: java.net.ProtocolException")); + break; + case BINARY: + case JSON: + case CMWLIGHT: + case UNKNOWN: + assertEquals(400, response.code()); + break; + default: + throw new IllegalStateException("test case not covered"); + } + } + + @ParameterizedTest + @EnumSource(value = MimeType.class, names = { "HTML", "JSON" }) + void testSet(final MimeType contentType) throws IOException { + final Request setRequest = new Request.Builder() // + .url("http://localhost:8080/helloWorld?noMenu") + .addHeader("accept", contentType.getMediaType()) + .post(new MultipartBody.Builder().setType(MultipartBody.FORM) // + .addFormDataPart("name", "needsName") + .addFormDataPart("customFilter", "myCustomName") + .addFormDataPart("byteReturnType", "1984") + .build()) + .build(); + final Response setResponse = okHttp.newCall(setRequest).execute(); + assertEquals(200, setResponse.code()); + + final Request getRequest = new Request.Builder().url("http://localhost:8080/helloWorld?noMenu").addHeader("accept", contentType.getMediaType()).get().build(); + final Response response = okHttp.newCall(getRequest).execute(); + final Headers header = response.headers(); + final String body = Objects.requireNonNull(response.body()).string(); + switch (contentType) { + case HTML: + assertThat(body, containsString("name=\"lsaContext\" value='myCustomName'")); + break; + case JSON: + assertThat(body, containsString("\"lsaContext\": \"myCustomName\",")); + break; + default: + throw new IllegalStateException("test case not covered"); + } + } + + @Test + void testSSE() { + AtomicInteger eventCounter = new AtomicInteger(); + Request request = new Request.Builder().url("http://localhost:8080/" + ImageService.PROPERTY_NAME).build(); + EventSourceListener eventSourceListener = new EventSourceListener() { + private final BlockingQueue events = new LinkedBlockingDeque<>(); + @Override + public void onEvent(final @NotNull EventSource eventSource, final String id, final String type, @NotNull String data) { + eventCounter.getAndIncrement(); + } + }; + final EventSource source = EventSources.createFactory(okHttp).newEventSource(request, eventSourceListener); + await().alias("wait for thread to start worker").atMost(1, TimeUnit.SECONDS).until(eventCounter::get, greaterThanOrEqualTo(3)); + assertThat(eventCounter.get(), greaterThanOrEqualTo(3)); + source.cancel(); + } + + private static OkHttpClient getUnsafeOkHttpClient() { + try { + // Create a trust manager that does not validate certificate chains + final TrustManager[] trustAllCerts = new TrustManager[] { + new X509TrustManager(){ + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType){} + + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType){} + + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers(){ + return new java.security.cert.X509Certificate[] {}; + } + } +}; + +// Install the all-trusting trust manager +final SSLContext sslContext = SSLContext.getInstance("SSL"); +sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); +// Create an ssl socket factory with our all-trusting manager +final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); + +OkHttpClient.Builder builder = new OkHttpClient.Builder(); +builder.sslSocketFactory(sslSocketFactory, (X509TrustManager) trustAllCerts[0]); +builder.hostnameVerifier((hostname, session) -> true); +return builder.build(); +} +catch (Exception e) { + throw new RuntimeException(e); +} +} +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/RestServerTests.java b/server-rest/src/test/java/io/opencmw/server/rest/RestServerTests.java new file mode 100644 index 00000000..6c1fe24d --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/RestServerTests.java @@ -0,0 +1,25 @@ +package io.opencmw.server.rest; + +import static io.opencmw.rbac.BasicRbacRole.ANYONE; + +import java.util.Collections; +import java.util.Set; + +import io.javalin.core.security.Role; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import io.opencmw.server.rest.user.RestUser; + +public class RestServerTests { + public static void main(String[] argv) { + Set accessRoles = Collections.singleton(new RestRole(ANYONE)); + + OpenApiDocumentation apiDocumentation = OpenApiBuilder.document().body(RestUser.class).json("200", RestUser.class); + RestServer.getInstance().get("/", OpenApiBuilder.documented(apiDocumentation, ctx -> { + ctx.html("Hello World!"); + }), accessRoles); + RestServer.getInstance().get("/helloWorld", OpenApiBuilder.documented(apiDocumentation, ctx -> { + ctx.html("Hello World!"); + }), accessRoles); + } +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/helper/ReplyDataType.java b/server-rest/src/test/java/io/opencmw/server/rest/helper/ReplyDataType.java new file mode 100644 index 00000000..efc0cc0e --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/helper/ReplyDataType.java @@ -0,0 +1,35 @@ +package io.opencmw.server.rest.helper; + +import io.opencmw.filter.TimingCtx; +import io.opencmw.serialiser.annotations.MetaInfo; + +import de.gsi.dataset.spi.utils.MultiArray; + +@MetaInfo(description = "reply type class description", direction = "OUT") +public class ReplyDataType { + @MetaInfo(description = "ReplyDataType name to show up in the OpenAPI docs") + public String name; + public boolean booleanReturnType; + public byte byteReturnType; + public short shortReturnType; + @MetaInfo(description = "a return value", unit = "A", direction = "OUT", groups = { "A", "B" }) + public int intReturnValue; + public long longReturnValue; + public byte[] byteArray; + public MultiArray multiArray = MultiArray.wrap(new double[] { 1.0, 2.0, 3.0 }, 0, new int[] { 1 }); + @MetaInfo(description = "WR timing context", direction = "OUT", groups = { "A", "B" }) + public TimingCtx timingCtx; + @MetaInfo(description = "LSA timing context", direction = "OUT", groups = { "A", "B" }) + public String lsaContext = ""; + @MetaInfo(description = "custom enum reply option", direction = "OUT", groups = { "A", "B" }) + public ReplyOption replyOption = ReplyOption.REPLY_OPTION2; + + public ReplyDataType() { + // needs default constructor + } + + @Override + public String toString() { + return "ReplyDataType{outputName='" + name + "', returnValue=" + intReturnValue + '}'; + } +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/helper/ReplyOption.java b/server-rest/src/test/java/io/opencmw/server/rest/helper/ReplyOption.java new file mode 100644 index 00000000..5d9a3457 --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/helper/ReplyOption.java @@ -0,0 +1,11 @@ +package io.opencmw.server.rest.helper; + +import io.opencmw.serialiser.annotations.MetaInfo; + +@MetaInfo(unit = "custom enum", description = "just for example purposes") +public enum ReplyOption { + REPLY_OPTION1, + REPLY_OPTION2, + REPLY_OPTION3, + REPLY_OPTION4 +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/helper/RequestDataType.java b/server-rest/src/test/java/io/opencmw/server/rest/helper/RequestDataType.java new file mode 100644 index 00000000..8e91bf2d --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/helper/RequestDataType.java @@ -0,0 +1,48 @@ +package io.opencmw.server.rest.helper; + +import io.opencmw.MimeType; +import io.opencmw.filter.TimingCtx; +import io.opencmw.serialiser.annotations.MetaInfo; + +@MetaInfo(description = "request type class description", direction = "IN") +public class RequestDataType { + @MetaInfo(description = " RequestDataType name to show up in the OpenAPI docs") + public String name = ""; + @MetaInfo(description = "FAIR timing context selector, e.g. FAIR.SELECTOR.C=0, ALL, ...") + public TimingCtx ctx = TimingCtx.get("FAIR.SELECTOR.ALL"); + @MetaInfo(description = "custom filter") + public String customFilter = ""; + @MetaInfo(description = "requested MIME content type, eg. 'application/binary', 'text/html','text/json', ..") + public MimeType contentType = MimeType.BINARY; + + public RequestDataType() { + // needs default constructor + } + + @Override + public String toString() { + return "RequestDataType{name='" + name + "', ctx=" + ctx + ", customFilter='" + customFilter + "'}"; + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (!(o instanceof RequestDataType)) + return false; + final RequestDataType that = (RequestDataType) o; + if (!name.equals(that.name)) + return false; + if (!ctx.equals(that.ctx)) + return false; + return customFilter.equals(that.customFilter); + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + ctx.hashCode(); + result = 31 * result + customFilter.hashCode(); + return result; + } +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/helper/TestContext.java b/server-rest/src/test/java/io/opencmw/server/rest/helper/TestContext.java new file mode 100644 index 00000000..589ff940 --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/helper/TestContext.java @@ -0,0 +1,23 @@ +package io.opencmw.server.rest.helper; + +import io.opencmw.MimeType; +import io.opencmw.filter.TimingCtx; +import io.opencmw.serialiser.annotations.MetaInfo; + +public class TestContext { + @MetaInfo(description = "FAIR timing context selector, e.g. FAIR.SELECTOR.C=0, ALL, ...") + public TimingCtx ctx = TimingCtx.get("FAIR.SELECTOR.ALL"); + @MetaInfo(unit = "a.u.", description = "random test parameter") + public String testFilter = "default value"; + @MetaInfo(description = "requested MIME content type, eg. 'application/binary', 'text/html','text/json', ..") + public MimeType contentType = MimeType.BINARY; + + public TestContext() { + // needs default constructor + } + + @Override + public String toString() { + return "TestContext{ctx=" + ctx + ", testFilter='" + testFilter + "', contentType=" + contentType + '}'; + } +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/samples/MajordomoRestPluginSample.java b/server-rest/src/test/java/io/opencmw/server/rest/samples/MajordomoRestPluginSample.java new file mode 100644 index 00000000..e8ed5332 --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/samples/MajordomoRestPluginSample.java @@ -0,0 +1,43 @@ +package io.opencmw.server.rest.samples; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.server.MajordomoBroker; +import io.opencmw.server.rest.MajordomoRestPlugin; +import io.opencmw.server.rest.test.HelloWorldService; +import io.opencmw.server.rest.test.ImageService; + +import zmq.util.Utils; + +public class MajordomoRestPluginSample { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoRestPluginSample.class); + + public static void main(String[] args) throws IOException { + MajordomoBroker primaryBroker = new MajordomoBroker("PrimaryBroker", "", BasicRbacRole.values()); + final String brokerRouterAddress = primaryBroker.bind("tcp://*:" + Utils.findOpenPort()); + primaryBroker.bind("mds://*:" + Utils.findOpenPort()); + MajordomoRestPlugin restPlugin = new MajordomoRestPlugin(primaryBroker.getContext(), "My test REST server", "*:8080", BasicRbacRole.ADMIN); + primaryBroker.start(); + restPlugin.start(); + LOGGER.atInfo().log("Broker and REST plugin started"); + + // start simple test services/properties + final HelloWorldService helloWorldService = new HelloWorldService(primaryBroker.getContext()); + helloWorldService.start(); + final ImageService imageService = new ImageService(primaryBroker.getContext(), 2000); + imageService.start(); + + // TODO: add OpenCMW client requesting binary and json models + + // second broker to test DNS functionalities + MajordomoBroker secondaryBroker = new MajordomoBroker("SecondaryTestBroker", brokerRouterAddress, BasicRbacRole.values()); + secondaryBroker.bind("tcp://*:" + Utils.findOpenPort()); + secondaryBroker.start(); + + LOGGER.atInfo().log("added services"); + } +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/test/HelloWorldService.java b/server-rest/src/test/java/io/opencmw/server/rest/test/HelloWorldService.java new file mode 100644 index 00000000..825a5725 --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/test/HelloWorldService.java @@ -0,0 +1,55 @@ +package io.opencmw.server.rest.test; + +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.ZContext; + +import io.opencmw.OpenCmwProtocol; +import io.opencmw.filter.TimingCtx; +import io.opencmw.rbac.RbacRole; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.server.MajordomoWorker; +import io.opencmw.server.rest.helper.ReplyDataType; +import io.opencmw.server.rest.helper.RequestDataType; +import io.opencmw.server.rest.helper.TestContext; + +@MetaInfo(unit = "short description", description = "This is an example property implementation.
" + + "Use this as a starting point for implementing your own properties
") +public class HelloWorldService extends MajordomoWorker { + private static final Logger LOGGER = LoggerFactory.getLogger(HelloWorldService.class); + private String customFilter = "uninitialised"; + public HelloWorldService(final ZContext ctx, final RbacRole... rbacRoles) { + super(ctx, "helloWorld", TestContext.class, RequestDataType.class, ReplyDataType.class, rbacRoles); + + this.setHandler((rawCtx, reqCtx, in, repCtx, out) -> { + // LOGGER.atInfo().addArgument(rawCtx).log("received rawCtx = {}") + LOGGER.atInfo().addArgument(reqCtx).log("received reqCtx = {}"); + LOGGER.atInfo().addArgument(in.name).log("received in.name = {}"); + + // some arbitrary data processing + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.UK); + out.name = "Hello World! The local time is: " + sdf.format(System.currentTimeMillis()); + out.byteArray = in.name.getBytes(StandardCharsets.UTF_8); + out.byteReturnType = 42; + out.timingCtx = TimingCtx.get("FAIR.SELECTOR.C=3"); + out.timingCtx.bpcts = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); + if (rawCtx.req.command == OpenCmwProtocol.Command.SET_REQUEST) { + // poor man's local setting management + customFilter = in.customFilter; + } + out.lsaContext = customFilter; + + repCtx.ctx = out.timingCtx; + repCtx.contentType = reqCtx.contentType; + repCtx.testFilter = "HelloWorld - reply topic = " + reqCtx.testFilter; + + LOGGER.atInfo().addArgument(repCtx).log("received repCtx = {}"); + LOGGER.atInfo().addArgument(out.name).log("received out.name = {}"); + }); + } +} diff --git a/server-rest/src/test/java/io/opencmw/server/rest/test/ImageService.java b/server-rest/src/test/java/io/opencmw/server/rest/test/ImageService.java new file mode 100644 index 00000000..0b7954cc --- /dev/null +++ b/server-rest/src/test/java/io/opencmw/server/rest/test/ImageService.java @@ -0,0 +1,76 @@ +package io.opencmw.server.rest.test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.ZContext; + +import io.opencmw.MimeType; +import io.opencmw.domain.BinaryData; +import io.opencmw.domain.NoData; +import io.opencmw.rbac.RbacRole; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.server.MajordomoWorker; +import io.opencmw.server.rest.helper.TestContext; + +import com.google.common.io.ByteStreams; + +@MetaInfo(unit = "test image service", description = "Simple test image service that rotates through" + + " a couple of pre-defined images @0.5 Hz") +public class ImageService extends MajordomoWorker { + private static final Logger LOGGER = LoggerFactory.getLogger(ImageService.class); + public static final String PROPERTY_NAME = "testImage"; + private static final String[] TEST_IMAGES = { "testimages/PM5544_test_signal.png", "testimages/SMPTE_Color_Bars.png" }; + private final byte[][] imageData; + private final AtomicInteger selectedImage = new AtomicInteger(); + + public ImageService(final ZContext ctx, final int updateInterval, final RbacRole... rbacRoles) { + super(ctx, PROPERTY_NAME, TestContext.class, NoData.class, BinaryData.class, rbacRoles); + imageData = new byte[TEST_IMAGES.length][]; + for (int i = 0; i < TEST_IMAGES.length; i++) { + try (final InputStream in = this.getClass().getResourceAsStream(TEST_IMAGES[i])) { + imageData[i] = ByteStreams.toByteArray(in); + LOGGER.atInfo().addArgument(TEST_IMAGES[i]).addArgument(imageData[i].length).log("read test image file: '{}' - bytes: {}"); + } catch (IOException e) { + LOGGER.atError().setCause(e).addArgument(TEST_IMAGES[i]).log("could not read test image file: '{}'"); + } + } + + final Timer timer = new Timer(); + timer.schedule(new TimerTask() { + @Override + public void run() { + selectedImage.set(selectedImage.incrementAndGet() % imageData.length); + final TestContext replyCtx = new TestContext(); + BinaryData reply = new BinaryData(); + reply.resourceName = "test.png"; + reply.data = imageData[selectedImage.get()]; //TODO rotate through images + replyCtx.contentType = MimeType.PNG; + ImageService.this.notify(replyCtx, reply); + // System.err.println("notify new image " + replyCtx) + } + }, TimeUnit.MILLISECONDS.toMillis(updateInterval), TimeUnit.MILLISECONDS.toMillis(updateInterval)); + + this.setHandler((rawCtx, reqCtx, in, repCtx, reply) -> { + final String path = StringUtils.stripStart(rawCtx.req.topic.getPath(), "/"); + LOGGER.atTrace().addArgument(reqCtx).addArgument(path).log("received reqCtx = {} - path='{}'"); + + reply.resourceName = StringUtils.stripStart(StringUtils.stripStart(path, PROPERTY_NAME), "/"); + reply.data = imageData[selectedImage.get()]; //TODO rotate through images + reply.contentType = MimeType.PNG; + + if (reply.resourceName.contains(".")) { + repCtx.contentType = reply.contentType; + } else { + repCtx.contentType = reqCtx.contentType; + } + }); + } +} diff --git a/server-rest/src/test/resources/io/opencmw/server/rest/test/testimages/PM5544_test_signal.png b/server-rest/src/test/resources/io/opencmw/server/rest/test/testimages/PM5544_test_signal.png new file mode 100644 index 00000000..4c55c9db Binary files /dev/null and b/server-rest/src/test/resources/io/opencmw/server/rest/test/testimages/PM5544_test_signal.png differ diff --git a/server-rest/src/test/resources/io/opencmw/server/rest/test/testimages/SMPTE_Color_Bars.png b/server-rest/src/test/resources/io/opencmw/server/rest/test/testimages/SMPTE_Color_Bars.png new file mode 100644 index 00000000..34d265e4 Binary files /dev/null and b/server-rest/src/test/resources/io/opencmw/server/rest/test/testimages/SMPTE_Color_Bars.png differ diff --git a/server-rest/src/test/resources/simplelogger.properties b/server-rest/src/test/resources/simplelogger.properties new file mode 100644 index 00000000..b4eba804 --- /dev/null +++ b/server-rest/src/test/resources/simplelogger.properties @@ -0,0 +1,50 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=info + +# The output target which can be the path to a file, or the special values "System.out" and "System.err". +# Default is "System.err". +org.slf4j.simpleLogger.logFile=System.out + +# If the output target is set to "System.out" or "System.err" (see preceding entry), by default, +# logs will be output to the latest value referenced by System.out/err variables. +# By setting this parameter to true, the output stream will be cached, i.e. assigned once at initialization +# time and re-used independently of the current value referenced by System.out/err. +org.slf4j.simpleLogger.cacheOutputStream=true + +# Logging detail level for a SimpleLogger instance named "a.b.c". Right-side value must be one of +# "trace", "debug", "info", "warn", "error" or "off". When a SimpleLogger named "a.b.c" is initialized, +# its level is assigned from this property. If unspecified, the level of nearest parent logger will be used, +# and if none is set, then the value specified by org.slf4j.simpleLogger.defaultLogLevel will be used. +org.slf4j.simpleLogger.log.io.opencmw.*=debug + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +# org.slf4j.simpleLogger.showThreadName=false + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +# org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/server/pom.xml b/server/pom.xml new file mode 100644 index 00000000..37e09d0b --- /dev/null +++ b/server/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + + io.opencmw + opencmw + ${revision}${sha1}${changelist} + ../pom.xml + + + server + + + ZeroMQ and REST-based micro-service server implementation. + + + + + io.opencmw + core + ${revision}${sha1}${changelist} + + + de.gsi.dataset + chartfx-dataset + ${version.chartfx} + test + + + + org.apache.velocity + velocity-engine-core + ${version.velocity} + compile + + + + diff --git a/server/src/main/java/io/opencmw/server/BasicMdpWorker.java b/server/src/main/java/io/opencmw/server/BasicMdpWorker.java new file mode 100644 index 00000000..73c97457 --- /dev/null +++ b/server/src/main/java/io/opencmw/server/BasicMdpWorker.java @@ -0,0 +1,377 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static io.opencmw.OpenCmwProtocol.*; +import static io.opencmw.OpenCmwProtocol.Command.*; +import static io.opencmw.OpenCmwProtocol.MdpMessage.receive; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; +import static io.opencmw.server.MajordomoBroker.SCHEME_MDP; +import static io.opencmw.server.MajordomoBroker.SCHEME_MDS; +import static io.opencmw.server.MajordomoBroker.SCHEME_TCP; +import static io.opencmw.utils.AnsiDefs.ANSI_RED; +import static io.opencmw.utils.AnsiDefs.ANSI_RESET; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.management.ManagementFactory; +import java.net.URI; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.rbac.RbacRole; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.serialiser.utils.ClassUtils; +import io.opencmw.utils.SystemProperties; + +/** + * Majordomo Protocol Client API, Java version Implements the OpenCmwProtocol/Worker spec at + * http://rfc.zeromq.org/spec:7. + * + *

+ * The worder is controlled by the following environment variables (see also MajordomoBroker definitions): + *

    + *
  • 'OpenCMW.heartBeat' [ms]: default (2500 ms) heart-beat time-out [ms]
    • + *
  • 'OpenCMW.heartBeatLiveness' []: default (3) heart-beat liveness - 3-5 is reasonable + * N.B. heartbeat expires when last heartbeat message is more than HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms ago. + * this implies also, that worker must either return their message within 'HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms' + * or decouple their secondary handler interface into another thread.
    • + *
+ * + */ +@MetaInfo(description = "default BasicMdpWorker implementation") +@SuppressWarnings({ "PMD.GodClass", "PMD.ExcessiveImports", "PMD.TooManyStaticImports", "PMD.DoNotUseThreads", "PMD.TooManyFields", "PMD.TooManyMethods" }) // makes the code more readable/shorter lines +public class BasicMdpWorker extends Thread { + protected static final byte[] RBAC = {}; //TODO: implement RBAC between Majordomo and Worker + protected static final String WILDCARD = "*"; + protected static final int HEARTBEAT_LIVENESS = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeatLiveness", 3); // [counts] 3-5 is reasonable + protected static final int HEARTBEAT_INTERVAL = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeat", 2500); // [ms] + protected static final AtomicInteger WORKER_COUNTER = new AtomicInteger(); + private static final Logger LOGGER = LoggerFactory.getLogger(BasicMdpWorker.class); + + static { + final String reason = "recursive definitions inside ZeroMQ"; + ClassUtils.DO_NOT_PARSE_MAP.put(ZContext.class, reason); + ClassUtils.DO_NOT_PARSE_MAP.put(ZMQ.Socket.class, reason); + ClassUtils.DO_NOT_PARSE_MAP.put(ZMQ.Poller.class, reason); + } + + // --------------------------------------------------------------------- + protected final String uniqueID; + protected final ZContext ctx; + protected final String brokerAddress; + protected final String serviceName; + protected final byte[] serviceBytes; + + protected final AtomicBoolean runSocketHandlerLoop = new AtomicBoolean(true); + protected final SortedSet rbacRoles; // NOSONAR NOPMD + protected final ZMQ.Socket notifySocket; // Socket to listener -- needed for thread-decoupling + protected final ZMQ.Socket notifyListenerSocket; // Socket to notifier -- needed for thread-decoupling + protected final List activeSubscriptions = Collections.synchronizedList(new ArrayList<>()); + protected final boolean isExternal; // used to skip heart-beating and disconnect checks + protected ZMQ.Socket workerSocket; // Socket to broker + protected ZMQ.Socket pubSocket; // Socket to broker + protected long heartbeatAt; // When to send HEARTBEAT + protected int liveness; // How many attempts left + protected long reconnect = 2500L; // Reconnect delay, msecs + protected RequestHandler requestHandler; + protected ZMQ.Poller poller; + + public BasicMdpWorker(String brokerAddress, String serviceName, final RbacRole... rbacRoles) { + this(null, brokerAddress, serviceName, rbacRoles); + } + + public BasicMdpWorker(ZContext ctx, String serviceName, final RbacRole... rbacRoles) { + this(ctx, "inproc://broker", serviceName, rbacRoles); + } + + protected BasicMdpWorker(ZContext ctx, String brokerAddress, String serviceName, final RbacRole... rbacRoles) { + super(); + assert (brokerAddress != null); + assert (serviceName != null); + this.brokerAddress = StringUtils.stripEnd(brokerAddress, "/"); + this.serviceName = StringUtils.stripStart(serviceName, "/"); + this.serviceBytes = this.serviceName.getBytes(UTF_8); + this.isExternal = !brokerAddress.toLowerCase(Locale.UK).contains("inproc://"); + + // initialise RBAC role-based priority queues + this.rbacRoles = Collections.unmodifiableSortedSet(new TreeSet<>(Set.of(rbacRoles))); + + this.ctx = Objects.requireNonNullElseGet(ctx, ZContext::new); + if (ctx != null) { + this.setDaemon(true); + } + this.setName(BasicMdpWorker.class.getSimpleName() + "#" + WORKER_COUNTER.getAndIncrement()); + this.uniqueID = this.serviceName + "-PID=" + ManagementFactory.getRuntimeMXBean().getName() + "-TID=" + this.getId(); + this.setName(this.getClass().getSimpleName() + "-" + uniqueID); + + notifyListenerSocket = this.ctx.createSocket(SocketType.PAIR); + notifyListenerSocket.bind("inproc://notifyListener" + uniqueID); + notifyListenerSocket.setHWM(0); + notifySocket = this.ctx.createSocket(SocketType.PAIR); + notifySocket.connect("inproc://notifyListener" + uniqueID); + notifySocket.setHWM(0); + + LOGGER.atTrace().addArgument(serviceName).addArgument(uniqueID).log("created new service '{}' worker - uniqueID: {}"); + } + + public SortedSet getRbacRoles() { // NOSONAR NOPMD + return rbacRoles; + } + + public Duration getReconnectDelay() { + return Duration.ofMillis(reconnect); + } + + public RequestHandler getRequestHandler() { + return requestHandler; + } + + public String getServiceName() { + return serviceName; + } + + public String getUniqueID() { + return uniqueID; + } + + /** + * Sends pre-defined message to subscriber (provided there is any that matches the published topic) + * @param notifyMessage the message that is supposed to be broadcast + * @return {@code false} in case message has not been sent (e.g. due to no pending subscriptions + */ + public boolean notify(@NotNull final MdpMessage notifyMessage) { + // send only if there are matching topics and duplicate messages based on topics as necessary + final URI originalTopic = notifyMessage.topic; + final List subTopics = new ArrayList<>(activeSubscriptions); // copy for decoupling/performance reasons + // N.B. for the time being only the path is matched - TODO: upgrade to full topic matching + if (subTopics.stream().filter(s -> s.startsWith(originalTopic.getPath()) || s.isBlank()).findFirst().isEmpty()) { + // block further processing of message + return false; + } + + notifyMessage.senderID = EMPTY_FRAME; + notifyMessage.protocol = PROT_WORKER; + notifyMessage.command = W_NOTIFY; + notifyMessage.serviceNameBytes = EMPTY_FRAME; + notifyMessage.clientRequestID = EMPTY_FRAME; + notifyMessage.topic = originalTopic; + return notifyRaw(notifyMessage); + } + + public BasicMdpWorker registerHandler(final RequestHandler requestHandler) { + this.requestHandler = requestHandler; + return this; + } + + /** + * primary run loop + * Send reply, if any, to broker and wait for next request. + */ + @Override + public void run() { + reconnectToBroker(); + // Poll socket for a reply, with timeout and/or until the process is stopped or interrupted + // N.B. poll(..) returns '-1' when thread is interrupted + final MdpMessage heartbeatMsg = new MdpMessage(null, PROT_WORKER, W_HEARTBEAT, serviceBytes, EMPTY_FRAME, EMPTY_URI, EMPTY_FRAME, "", RBAC); + while (runSocketHandlerLoop.get() && !Thread.currentThread().isInterrupted() && poller.poll(HEARTBEAT_INTERVAL) != -1) { + boolean dataReceived = true; + while (dataReceived) { + // handle message from or to broker + final MdpMessage brokerMsg = receive(workerSocket, false); + dataReceived = MdpMessage.send(workerSocket, handleRequestsFromBroker(brokerMsg)); + + final ZMsg pubMsg = ZMsg.recvMsg(pubSocket, false); + dataReceived |= handleSubscriptionMsg(pubMsg); + + // handle message from or to notify thread + final MdpMessage notifyMsg = receive(notifyListenerSocket, false); + if (notifyMsg != null) { + // forward notify message to MDP broker + dataReceived |= notifyMsg.send(workerSocket); + } + } + + if (System.currentTimeMillis() > heartbeatAt && --liveness == 0) { + LOGGER.atWarn().addArgument(uniqueID).log("worker '{}' disconnected from broker - retrying"); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(reconnect)); + reconnectToBroker(); + } + + // Send HEARTBEAT if it's time + if (System.currentTimeMillis() > heartbeatAt) { + heartbeatMsg.send(workerSocket); + heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; + } + } + if (Thread.currentThread().isInterrupted()) { + LOGGER.atWarn().addArgument(uniqueID).log("worker '{}' interrupt received, killing worker"); + } + if (isExternal) { + ctx.destroy(); + } + } + + public void setReconnectDelay(final int reconnect, @NotNull final TimeUnit timeUnit) { + this.reconnect = timeUnit.toMillis(reconnect); + } + + @Override + public synchronized void start() { // NOPMD 'synchronized' comes from JDK class definition + runSocketHandlerLoop.set(true); + super.start(); + } + + public void stopWorker() { + runSocketHandlerLoop.set(false); + } + + protected List handleRequestsFromBroker(final MdpMessage request) { + if (request == null) { + return Collections.emptyList(); + } + + liveness = HEARTBEAT_LIVENESS; + + switch (request.command) { + case GET_REQUEST: + case SET_REQUEST: + case W_NOTIFY: + case PARTIAL: + case FINAL: + return processRequest(request); + case W_HEARTBEAT: + // Do nothing for heartbeats + return Collections.emptyList(); + case DISCONNECT: + // TODO: check whether to reconnect or to disconnect permanently + reconnectToBroker(); + return Collections.emptyList(); + case READY: + case SUBSCRIBE: + case UNSUBSCRIBE: + case UNKNOWN: + // N.B. not too verbose logging since we do not want that sloppy clients + // can bring down the broker through warning or info messages + if (LOGGER.isDebugEnabled()) { + LOGGER.atWarn().addArgument(getServiceName()).addArgument(request.command).log("service '{}' erroneously received {} command - should be handled in Majordomo broker"); + } + return Collections.emptyList(); + default: + } + throw new IllegalStateException("should not reach here - request message = " + request); + } + + protected boolean handleSubscriptionMsg(final ZMsg subMsg) { // NOPMD + if (subMsg == null || subMsg.isEmpty()) { + return false; + } + final byte[] topicBytes = subMsg.getFirst().getData(); + if (topicBytes.length == 0) { + return false; + } + final Command subType = topicBytes[0] == 1 ? SUBSCRIBE : (topicBytes[0] == 0 ? UNSUBSCRIBE : UNKNOWN); // '1'('0' being the default ZeroMQ (un-)subscribe command + final String subscriptionTopic = new String(topicBytes, 1, topicBytes.length - 1, UTF_8); + if (LOGGER.isDebugEnabled() && (subscriptionTopic.isBlank() || subscriptionTopic.contains(getServiceName()))) { + LOGGER.atDebug().addArgument(getServiceName()).addArgument(subType).addArgument(subscriptionTopic).log("Service '{}' received subscription request: {} to '{}'"); + } + + if (!subscriptionTopic.isBlank() && !subscriptionTopic.startsWith(getServiceName())) { + // subscription topic for another service + return false; + } + switch (subType) { + case SUBSCRIBE: + activeSubscriptions.add(subscriptionTopic); + return true; + case UNSUBSCRIBE: + activeSubscriptions.remove(subscriptionTopic); + return true; + case UNKNOWN: + default: + return false; + } + } + + protected boolean notifyRaw(@NotNull final MdpMessage notifyMessage) { + assert notifyMessage != null : "notify message must not be null"; + return notifyMessage.send(notifySocket); + } + + protected List processRequest(final MdpMessage request) { + // de-serialise byte[] -> PropertyMap() (+ getObject(Class)) + try { + final Context mdpCtx = new Context(request); + getRequestHandler().handle(mdpCtx); + return mdpCtx.rep == null ? Collections.emptyList() : List.of(mdpCtx.rep); + } catch (Throwable e) { // NOPMD on purpose since we want to catch exceptions and courteously return this to the user + final StringWriter sw = new StringWriter(); + final PrintWriter pw = new PrintWriter(sw); + e.printStackTrace(pw); + final String exceptionMsg = ANSI_RED + getClass().getName() + " caught exception for service '" + getServiceName() + + "'\nrequest msg: " + request + "\nexception: " + sw.toString() + ANSI_RESET; + if (LOGGER.isDebugEnabled()) { + LOGGER.atError().addArgument(exceptionMsg).log("could not processRequest(MdpMessage) - exception thrown:\n{}"); + } + return List.of(new MdpMessage(request.senderID, request.protocol, FINAL, request.serviceNameBytes, request.clientRequestID, request.topic, null, exceptionMsg, RBAC)); + } + } + + /** + * Connect or reconnect to broker + */ + protected void reconnectToBroker() { + if (workerSocket != null) { + workerSocket.close(); + } + final String translatedBrokerAddress = brokerAddress.replace(SCHEME_MDP, SCHEME_TCP).replace(SCHEME_MDS, SCHEME_TCP); + workerSocket = ctx.createSocket(SocketType.DEALER); + assert workerSocket != null : "worker socket is null"; + workerSocket.setHWM(0); + workerSocket.connect(translatedBrokerAddress + MajordomoBroker.SUFFIX_ROUTER); + + if (pubSocket != null) { + pubSocket.close(); + } + pubSocket = ctx.createSocket(SocketType.XPUB); + assert pubSocket != null : "publication socket is null"; + pubSocket.setHWM(0); + pubSocket.setXpubVerbose(true); + pubSocket.connect(translatedBrokerAddress + MajordomoBroker.SUFFIX_SUBSCRIBE); + + // Register service with broker + LOGGER.atInfo().addArgument(brokerAddress).log("register service with broker '{}"); + final byte[] classNameByte = this.getClass().getName().getBytes(UTF_8); // used for OpenAPI purposes + new MdpMessage(null, PROT_WORKER, READY, serviceBytes, EMPTY_FRAME, URI.create(serviceName), classNameByte, "", RBAC).send(workerSocket); + + if (poller != null) { + poller.unregister(workerSocket); + poller.close(); + } + poller = ctx.createPoller(3); + poller.register(workerSocket, ZMQ.Poller.POLLIN); + poller.register(pubSocket, ZMQ.Poller.POLLIN); + poller.register(notifyListenerSocket, ZMQ.Poller.POLLIN); + + // If liveness hits zero, queue is considered disconnected + liveness = HEARTBEAT_LIVENESS; + heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; + } + + public interface RequestHandler { + void handle(Context ctx) throws Throwable; // NOPMD NOSONAR - should allow to throw any/generic exceptions + } +} diff --git a/server/src/main/java/io/opencmw/server/DefaultHtmlHandler.java b/server/src/main/java/io/opencmw/server/DefaultHtmlHandler.java new file mode 100644 index 00000000..7f089d1f --- /dev/null +++ b/server/src/main/java/io/opencmw/server/DefaultHtmlHandler.java @@ -0,0 +1,119 @@ +package io.opencmw.server; + +import java.io.StringWriter; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +import org.apache.commons.lang3.StringUtils; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; +import org.zeromq.util.ZData; + +import io.opencmw.OpenCmwProtocol; +import io.opencmw.QueryParameterParser; +import io.opencmw.domain.BinaryData; +import io.opencmw.serialiser.FieldDescription; +import io.opencmw.serialiser.spi.ClassFieldDescription; +import io.opencmw.serialiser.utils.ClassUtils; + +public class DefaultHtmlHandler implements MajordomoWorker.Handler { + public static final String NO_MENU = "noMenu"; + private static final String TEMPLATE_DEFAULT = "/velocity/property/defaultPropertyLayout.vm"; + protected final Class mdpWorkerClass; + protected final Consumer> userContextMapModifier; + protected final String velocityTemplate; + protected final VelocityEngine velocityEngine = new VelocityEngine(); + + public DefaultHtmlHandler(final Class mdpWorkerClass, final String velocityTemplate, final Consumer> userContextMapModifier) { + this.mdpWorkerClass = mdpWorkerClass; + this.velocityTemplate = velocityTemplate == null ? TEMPLATE_DEFAULT : velocityTemplate; + this.userContextMapModifier = userContextMapModifier; + velocityEngine.setProperty("resource.loaders", "class"); + velocityEngine.setProperty("resource.loader.class.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader"); + velocityEngine.setProperty("velocimacro.library.autoreload", "true"); + velocityEngine.setProperty("resource.loader.file.cache", "false"); + velocityEngine.setProperty("velocimacro.inline.replace_global", "true"); + velocityEngine.init(); + } + + @Override + public void handle(OpenCmwProtocol.Context rawCtx, C requestCtx, I request, C replyCtx, O reply) { + final String queryString = rawCtx.req.topic.getQuery(); + final boolean noMenu = queryString != null && queryString.contains(NO_MENU); + + final HashMap context = new HashMap<>(); + // pre-fill context + + context.put(NO_MENU, noMenu); + try { + context.put("requestedURI", rawCtx.req.topic.toString()); + context.put("requestedURInoFrame", QueryParameterParser.appendQueryParameter(rawCtx.req.topic, NO_MENU).toString()); + } catch (URISyntaxException e) { + throw new IllegalStateException("appendURI error for " + rawCtx.req.topic, e); + } + final ClassFieldDescription fieldDescription = mdpWorkerClass == null ? null : ClassUtils.getFieldDescription(mdpWorkerClass); + context.put("service", StringUtils.stripStart(rawCtx.req.topic.getPath(), "/")); + context.put("mdpClass", mdpWorkerClass); + context.put("mdpMetaData", fieldDescription); + context.put("mdpCommand", rawCtx.req.command); + context.put("clientRequestID", ZData.toString(rawCtx.req.clientRequestID)); + + context.put("requestTopic", rawCtx.req.topic); + context.put("replyTopic", rawCtx.rep.topic); + context.put("requestCtx", requestCtx); + context.put("replyCtx", replyCtx); + context.put("request", request); + context.put("reply", reply); + context.put("requestCtxClassData", generateQueryParameter(requestCtx)); + context.put("replyCtxClassData", generateQueryParameter(replyCtx)); + + context.put("requestClassData", generateQueryParameter(request)); + if (BinaryData.class.equals(request.getClass())) { + final byte[] rawData = ((BinaryData) request).data; + context.put("requestMimeType", ((BinaryData) reply).contentType.toString()); + context.put("requestResourceName", ((BinaryData) reply).resourceName); + context.put("requestRawData", Base64.getEncoder().encodeToString(rawData)); + } + + context.put("replyClassData", generateQueryParameter(reply)); + if (BinaryData.class.equals(reply.getClass())) { + final byte[] rawData = ((BinaryData) reply).data; + context.put("replyMimeType", ((BinaryData) reply).contentType.toString()); + context.put("replyResourceName", ((BinaryData) reply).resourceName); + context.put("replyRawData", Base64.getEncoder().encodeToString(rawData)); + } + + if (userContextMapModifier != null) { + userContextMapModifier.accept(context); + } + + StringWriter writer = new StringWriter(); + velocityEngine.getTemplate(velocityTemplate).merge(new VelocityContext(context), writer); + String returnVal = writer.toString(); + rawCtx.rep.data = returnVal.getBytes(StandardCharsets.UTF_8); + } + + public static Map generateQueryParameter(Object obj) { + final ClassFieldDescription fieldDescription = ClassUtils.getFieldDescription(obj.getClass()); + final Map map = new HashMap<>(); // NOPMD - no concurrent access, used in a single thread and is then destroyed + final List children = fieldDescription.getChildren(); + for (FieldDescription child : children) { + ClassFieldDescription field = (ClassFieldDescription) child; + final BiFunction mapFunction = QueryParameterParser.CLASS_TO_STRING_CONVERTER.get(field.getType()); + final String str; + if (mapFunction == null) { + str = QueryParameterParser.CLASS_TO_STRING_CONVERTER.get(Object.class).apply(obj, field); + } else { + str = mapFunction.apply(obj, field); + } + map.put(field, StringUtils.stripEnd(StringUtils.stripStart(str, "\"["), "\"]")); + } + return map; + } +} diff --git a/server/src/main/java/io/opencmw/server/MajordomoBroker.java b/server/src/main/java/io/opencmw/server/MajordomoBroker.java new file mode 100644 index 00000000..66426706 --- /dev/null +++ b/server/src/main/java/io/opencmw/server/MajordomoBroker.java @@ -0,0 +1,916 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static org.zeromq.ZMQ.Socket; +import static org.zeromq.util.ZData.strhex; + +import static io.opencmw.OpenCmwProtocol.*; +import static io.opencmw.OpenCmwProtocol.Command.*; +import static io.opencmw.OpenCmwProtocol.MdpMessage.receive; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_CLIENT; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; +import static io.opencmw.server.MmiServiceHelper.*; + +import java.net.DatagramSocket; +import java.net.InetAddress; +import java.net.SocketException; +import java.net.URI; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; +import org.zeromq.util.ZData; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacRole; +import io.opencmw.rbac.RbacToken; +import io.opencmw.utils.NoDuplicatesList; +import io.opencmw.utils.SystemProperties; + +/** + * Majordomo Protocol broker -- a minimal implementation of http://rfc.zeromq.org/spec:7 and spec:8 and following the OpenCMW specification + * + *

+ * The broker is controlled by the following environment variables: + *

    + *
  • 'OpenCMW.heartBeat' [ms]: default (2500 ms) heart-beat time-out [ms]
    • + *
  • 'OpenCMW.heartBeatLiveness' []: default (3) heart-beat liveness - 3-5 is reasonable + * N.B. heartbeat expires when last heartbeat message is more than HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms ago. + * this implies also, that worker must either return their message within 'HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS ms' + * or decouple their secondary handler interface into another thread.
    • + * + *
  • 'OpenCMW.clientTimeOut' [s]: default (3600, i.e. 1h) time-out after which unanswered client messages/infos are being deleted
    • + *
  • 'OpenCMW.nIoThreads' []: default (2) IO threads dedicated to network IO (ZeroMQ recommendation 1 thread per 1 GBit/s)
    • + *
  • 'OpenCMW.dnsTimeOut' [s]: default (60) DNS time-out after which an unresponsive client is dropped from the DNS table + * N.B. if registered, a HEARTBEAT challenge will be send that needs to be replied with a READY command/re-registering
    • + *
+ */ +@SuppressWarnings({ "PMD.DefaultPackage", "PMD.UseConcurrentHashMap", "PMD.TooManyFields", "PMD.TooManyMethods", "PMD.TooManyStaticImports", "PMD.CommentSize", "PMD.UseConcurrentHashMap" }) // package private explicitly needed for MmiServiceHelper, thread-safe/performance use of HashMap +public class MajordomoBroker extends Thread { + public static final byte[] RBAC = {}; // TODO: implement RBAC between Majordomo and Worker + // ----------------- default service names ----------------------------- + public static final String SUFFIX_ROUTER = "/router"; + public static final String SUFFIX_PUBLISHER = "/publisher"; + public static final String SUFFIX_SUBSCRIBE = "/subscribe"; + public static final String INPROC_BROKER = "inproc://broker"; + public static final String INTERNAL_ADDRESS_BROKER = INPROC_BROKER + SUFFIX_ROUTER; + public static final String INTERNAL_ADDRESS_PUBLISHER = INPROC_BROKER + SUFFIX_PUBLISHER; + public static final String INTERNAL_ADDRESS_SUBSCRIBE = INPROC_BROKER + SUFFIX_SUBSCRIBE; + public static final String SCHEME_HTTP = "http://"; + public static final String SCHEME_HTTPS = "https://"; + public static final String SCHEME_MDP = "mdp://"; + public static final String SCHEME_MDS = "mds://"; + public static final String SCHEME_TCP = "tcp://"; + public static final String WILDCARD = "*"; + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoBroker.class); + private static final long HEARTBEAT_LIVENESS = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeatLiveness", 3); // [counts] 3-5 is reasonable + private static final long HEARTBEAT_INTERVAL = SystemProperties.getValueIgnoreCase("OpenCMW.heartBeat", 2500); // [ms] + private static final long HEARTBEAT_EXPIRY = HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS; + private static final long CLIENT_TIMEOUT = TimeUnit.SECONDS.toMillis(SystemProperties.getValueIgnoreCase("OpenCMW.clientTimeOut", 0)); // [s] + private static final int N_IO_THREAD = SystemProperties.getValueIgnoreCase("OpenCMW.nIoThreads", 1); // [] typ. 1 for < 1 GBit/s + private static final long DNS_TIMEOUT = TimeUnit.SECONDS.toMillis(SystemProperties.getValueIgnoreCase("OpenCMW.dnsTimeOut", 10)); // [ms] time when + private static final AtomicInteger BROKER_COUNTER = new AtomicInteger(); + // --------------------------------------------------------------------- + protected final ZContext ctx; + protected final Socket routerSocket; + protected final Socket pubSocket; + protected final Socket subSocket; + protected final Socket dnsSocket; + protected final String brokerName; + protected final String dnsAddress; + protected final List routerSockets = new NoDuplicatesList<>(); // Sockets for clients & public external workers + protected final SortedSet> rbacRoles; + /* default */ final Map services = new HashMap<>(); // known services Map<'service name', Service> + protected final Map workers = new HashMap<>(); // known workers Map clients = new HashMap<>(); + protected final Map activeSubscriptions = new HashMap<>(); // Map> + protected final Map> routerBasedSubscriptions = new HashMap<>(); // Map> + private final AtomicBoolean run = new AtomicBoolean(false); // NOPMD - nomen est omen + private final Deque waiting = new ArrayDeque<>(); // idle workers + /* default */ final Map dnsCache = new HashMap<>(); // + private long heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; // When to send HEARTBEAT + private long dnsHeartbeatAt = System.currentTimeMillis() + DNS_TIMEOUT; // When to send a DNS HEARTBEAT + + /** + * Initialize broker state. + * + * @param brokerName specific Majordomo Broker name this instance is known for in the world + * @param dnsAddress specifc of other Majordomo broker that acts as primary DNS + * @param rbacRoles RBAC-based roles (used for IO prioritisation and service access control + */ + public MajordomoBroker(@NotNull final String brokerName, @NotNull final String dnsAddress, final RbacRole... rbacRoles) { + super(); + this.brokerName = brokerName; + final URI dnsService = URI.create(dnsAddress); + this.dnsAddress = dnsAddress.isBlank() ? "" : SCHEME_TCP + dnsService.getAuthority() + dnsService.getPath(); + this.setName(MajordomoBroker.class.getSimpleName() + "(" + brokerName + ")#" + BROKER_COUNTER.getAndIncrement()); + + ctx = new ZContext(N_IO_THREAD); + + // initialise RBAC role-based priority queues + this.rbacRoles = Collections.unmodifiableSortedSet(new TreeSet<>(Set.of(rbacRoles))); + + // generate and register internal default inproc socket + routerSocket = ctx.createSocket(SocketType.ROUTER); + routerSocket.setHWM(0); + routerSocket.bind(INTERNAL_ADDRESS_BROKER); // NOPMD + pubSocket = ctx.createSocket(SocketType.XPUB); + pubSocket.setHWM(0); + pubSocket.setXpubVerbose(true); + pubSocket.bind(INTERNAL_ADDRESS_PUBLISHER); // NOPMD + subSocket = ctx.createSocket(SocketType.SUB); + subSocket.setHWM(0); + subSocket.bind(INTERNAL_ADDRESS_SUBSCRIBE); // NOPMD + + registerDefaultServices(rbacRoles); // NOPMD + + dnsSocket = ctx.createSocket(SocketType.DEALER); + dnsSocket.setHWM(0); + if (this.dnsAddress.isBlank()) { + dnsSocket.connect(INTERNAL_ADDRESS_BROKER); + } else { + dnsSocket.connect(this.dnsAddress); + } + + LOGGER.atInfo().addArgument(getName()).addArgument(this.dnsAddress).log("register new '{}' broker with DNS: '{}'"); + } + + /** + * Add internal service. + * + * @param worker the worker + */ + public void addInternalService(final BasicMdpWorker worker) { + assert worker != null : "worker must not be null"; + requireService(worker.getServiceName(), worker); + } + + /** + * Bind broker to endpoint, can call this multiple times. We use a single + * socket for both clients and workers. + *

+ * + * @param endpoint the URI-based 'scheme://ip:port' endpoint definition the server should listen to

The protocol definition

  • 'mdp://' corresponds to a SocketType.ROUTER socket
  • 'mds://' corresponds to a SocketType.XPUB socket
  • 'tcp://' internally falls back to 'mdp://' and ROUTER socket
+ * @return the string + */ + public String bind(String endpoint) { + final boolean isRouterSocket = !endpoint.startsWith(SCHEME_MDS); + final String endpointAdjusted; + if (isRouterSocket) { + routerSocket.bind(endpoint.replace(SCHEME_MDP, SCHEME_TCP)); + endpointAdjusted = endpoint.replace(SCHEME_TCP, SCHEME_MDP); + } else { + pubSocket.bind(endpoint.replace(SCHEME_MDS, SCHEME_TCP)); + endpointAdjusted = endpoint.replace(SCHEME_TCP, SCHEME_MDS); + } + final String adjustedAddressPublic = endpointAdjusted.replace(WILDCARD, getLocalHostName()); + routerSockets.add(adjustedAddressPublic); + if (endpoint.contains(WILDCARD)) { + routerSockets.add(endpointAdjusted.replace(WILDCARD, "localhost")); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(adjustedAddressPublic).log("Majordomo broker/0.1 is active at '{}'"); + } + return adjustedAddressPublic; + } + + public ZContext getContext() { + return ctx; + } + + public Socket getInternalRouterSocket() { + return routerSocket; + } + + /** + * Gets router sockets. + * + * @return unmodifiable list of registered external sockets + */ + public List getRouterSockets() { + return Collections.unmodifiableList(routerSockets); + } + + public Collection getServices() { + return services.values(); + } + + public boolean isRunning() { + return run.get(); + } + + public void removeService(final String serviceName) { + final Service ret = services.remove(serviceName); + ret.mdpWorker.forEach(BasicMdpWorker::stopWorker); + ret.waiting.forEach(worker -> new MdpMessage(worker.address, PROT_WORKER, DISCONNECT, worker.service.nameBytes, EMPTY_FRAME, URI.create(worker.service.name), EMPTY_FRAME, "", RBAC).send(worker.socket)); + } + + /** + * main broker work happens here + */ + @Override + public void run() { + try (ZMQ.Poller items = ctx.createPoller(4)) { // 4 -> four sockets defined below + items.register(routerSocket, ZMQ.Poller.POLLIN); + items.register(dnsSocket, ZMQ.Poller.POLLIN); + items.register(pubSocket, ZMQ.Poller.POLLIN); + items.register(subSocket, ZMQ.Poller.POLLIN); + while (run.get() && !Thread.currentThread().isInterrupted() && items.poll(HEARTBEAT_INTERVAL) != -1) { + int loopCount = 0; + boolean receivedMsg = true; + while (run.get() && !Thread.currentThread().isInterrupted() && receivedMsg) { + final MdpMessage routerMsg = receive(routerSocket, false); + receivedMsg = handleReceivedMessage(routerSocket, routerMsg); + + final MdpMessage subMsg = receive(subSocket, false); + receivedMsg |= handleReceivedMessage(subSocket, subMsg); + + final MdpMessage dnsMsg = receive(dnsSocket, false); + receivedMsg |= handleReceivedMessage(dnsSocket, dnsMsg); + + final ZMsg pubMsg = ZMsg.recvMsg(pubSocket, false); + receivedMsg |= handleSubscriptionMsg(pubMsg); + + processClients(); + if (loopCount % 10 == 0) { + // perform maintenance tasks during the first and every tenth + // iteration + purgeWorkers(); + purgeClients(); + purgeDnsServices(); + sendHeartbeats(); + sendDnsHeartbeats(false); + } + loopCount++; + } + } + } + destroy(); // interrupted + } + + private boolean handleSubscriptionMsg(final ZMsg subMsg) { + if (subMsg == null || subMsg.isEmpty()) { + return false; + } + final byte[] topicBytes = subMsg.getFirst().getData(); + if (topicBytes.length == 0) { + return false; + } + final Command subType = topicBytes[0] == 1 ? SUBSCRIBE : (topicBytes[0] == 0 ? UNSUBSCRIBE : UNKNOWN); // '1'('0' being the default ZeroMQ (un-)subscribe command + final String subscriptionTopic = new String(topicBytes, 1, topicBytes.length - 1, UTF_8); + LOGGER.atDebug().addArgument(subType).addArgument(subscriptionTopic).log("received subscription request: {} to '{}'"); + + switch (subType) { + case SUBSCRIBE: + if (activeSubscriptions.computeIfAbsent(subscriptionTopic, s -> new AtomicInteger()).incrementAndGet() == 1) { + subSocket.subscribe(subscriptionTopic); + } + return true; + case UNSUBSCRIBE: + if (activeSubscriptions.computeIfAbsent(subscriptionTopic, s -> new AtomicInteger()).decrementAndGet() <= 0) { + subSocket.unsubscribe(subscriptionTopic); + } + return true; + case UNKNOWN: + default: + return false; + } + } + + @Override + public synchronized void start() { // NOPMD - need to be synchronised on class level due to super definition + run.set(true); + services.forEach((serviceName, service) -> service.internalWorkers.forEach(Thread::start)); + super.start(); + sendDnsHeartbeats(true); // initial register of default routes + } + + /** + * Stop broker. + */ + public void stopBroker() { + run.set(false); + } + + /** + * Deletes worker from all data structures, and destroys worker. + * + * @param worker internal reference to worker + * @param disconnect true: send a disconnect message to worker + */ + protected void deleteWorker(Worker worker, boolean disconnect) { + assert (worker != null); + if (disconnect) { + new MdpMessage(worker.address, PROT_WORKER, DISCONNECT, + worker.serviceName, EMPTY_FRAME, + URI.create(new String(worker.serviceName, UTF_8)), EMPTY_FRAME, "", RBAC) + .send(worker.socket); + } + if (worker.service != null) { + worker.service.waiting.remove(worker); + } + workers.remove(worker.addressHex); + } + + /** + * Disconnect all workers, destroy context. + */ + protected void destroy() { + Worker[] deleteList = workers.values().toArray(new Worker[0]); + for (Worker worker : deleteList) { + deleteWorker(worker, true); + } + ctx.destroy(); + } + + /** + * Dispatch requests to waiting workers as possible + * + * @param service dispatch message for this service + */ + protected void dispatch(Service service) { + assert (service != null); + purgeWorkers(); + while (!service.waiting.isEmpty() && service.requestsPending()) { + final MdpMessage msg = service.getNextPrioritisedMessage(); + if (msg == null) { + // should be thrown only with VM '-ea' enabled -- assert noisily since + // this a (rare|design) library error + assert false : "getNextPrioritisedMessage should not be null"; + continue; + } + Worker worker = service.waiting.pop(); + waiting.remove(worker); + msg.serviceNameBytes = msg.senderID; + msg.senderID = worker.address; // replace sourceID with worker destinationID + msg.protocol = PROT_WORKER; // CLIENT protocol -> WORKER -> protocol + msg.send(worker.socket); + } + } + + /** + * Handle received message boolean. + * + * @param receiveSocket the receive socket + * @param msg the to be processed msg + * @return true if request was implemented and has been processed + */ + protected boolean handleReceivedMessage(final Socket receiveSocket, final MdpMessage msg) { + if (msg == null) { + return false; + } + final String topic = msg.topic.toString(); + switch (msg.protocol) { + case PROT_CLIENT: + case PROT_CLIENT_HTTP: + // Set reply return address to client sender + switch (msg.command) { + case READY: + if (msg.topic.getScheme() != null) { + // register potentially new service + DnsServiceItem ret = dnsCache.computeIfAbsent(msg.getServiceName(), s -> new DnsServiceItem(msg.senderID, msg.getServiceName())); + ret.uri.add(msg.topic); + ret.updateExpiryTimeStamp(); + } + return true; + case SUBSCRIBE: + if (activeSubscriptions.computeIfAbsent(topic, s -> new AtomicInteger()).incrementAndGet() == 1) { + subSocket.subscribe(topic); + } + routerBasedSubscriptions.computeIfAbsent(topic, s -> new ArrayList<>()).add(msg.senderID); + return true; + case UNSUBSCRIBE: + if (activeSubscriptions.computeIfAbsent(topic, s -> new AtomicInteger()).decrementAndGet() <= 0) { + subSocket.unsubscribe(topic); + } + routerBasedSubscriptions.computeIfAbsent(topic, s -> new ArrayList<>()).remove(msg.senderID); + if (routerBasedSubscriptions.get(topic).isEmpty()) { + routerBasedSubscriptions.remove(topic); + } + return true; + case W_HEARTBEAT: + sendDnsHeartbeats(true); + return true; + default: + } + + final String senderName = msg.getSenderName(); + final Client client = clients.computeIfAbsent(senderName, s -> new Client(receiveSocket, senderName, msg.senderID)); + client.offerToQueue(msg); + return true; + case PROT_WORKER: + processWorker(receiveSocket, msg); + return true; + default: + // N.B. not too verbose logging since we do not want that sloppy clients + // can bring down the broker through warning or info messages + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(msg).log("Majordomo broker invalid message: '{}'"); + } + return false; + } + } + + /** + * Process a request coming from a client. + */ + protected void processClients() { + // round-robbin + clients.forEach((name, client) -> { + final MdpMessage clientMessage = client.pop(); + if (clientMessage == null) { + return; + } + + // dispatch client message to worker queue + // old : final Service service = services.get(clientMessage.getServiceName()) + final Service service = getBestMatchingService(clientMessage.getServiceName()); + if (service == null) { + // not implemented -- according to Majordomo Management Interface (MMI) + // as defined in http://rfc.zeromq.org/spec:8 + new MdpMessage(clientMessage.senderID, PROT_CLIENT, FINAL, + clientMessage.serviceNameBytes, + clientMessage.clientRequestID, + URI.create(INTERNAL_SERVICE_NAMES), + "501".getBytes(UTF_8), "unknown service (error 501): '" + clientMessage.getServiceName() + '\'', RBAC) + .send(client.socket); + return; + } + // queue new client message RBAC-priority-based + service.putPrioritisedMessage(clientMessage); + + // dispatch service + dispatch(service); + }); + } + + Service getBestMatchingService(final String serviceName) { // NOPMD package private OK + final List sortedList = services.keySet().stream().filter(serviceName::startsWith).sorted(Comparator.comparingInt(String::length)).collect(Collectors.toList()); + return sortedList.isEmpty() ? null : services.get(sortedList.get(0)); + } + + /** + * Process message sent to us by a worker. + * + * @param receiveSocket the socket the message was received at + * @param msg the received and to be processed message + */ + protected void processWorker(final Socket receiveSocket, final MdpMessage msg) { //NOPMD + final String senderIdHex = strhex(msg.senderID); + final String serviceName = msg.getServiceName(); + final boolean workerReady = workers.containsKey(senderIdHex); + final Worker worker = requireWorker(receiveSocket, msg.senderID, senderIdHex, msg.serviceNameBytes); + + switch (msg.command) { + case READY: + LOGGER.atTrace().addArgument(serviceName).log("log new local/external worker for service {} - " + msg); + // Attach worker to service and mark as idle + worker.service = requireService(serviceName); + workerWaiting(worker); + worker.service.serviceDescription = Arrays.copyOf(msg.data, msg.data.length); + + if (!msg.topic.toString().isBlank() && msg.topic.getScheme() != null) { + routerSockets.add(msg.topic.toString()); + DnsServiceItem ret = dnsCache.computeIfAbsent(brokerName, s -> new DnsServiceItem(msg.senderID, brokerName)); + ret.uri.add(msg.topic); + } + + // notify potential listener + msg.data = msg.serviceNameBytes; + msg.serviceNameBytes = INTERNAL_SERVICE_NAMES.getBytes(UTF_8); + msg.command = W_NOTIFY; + msg.clientRequestID = this.getName().getBytes(UTF_8); + msg.topic = URI.create(INTERNAL_SERVICE_NAMES); + msg.errors = ""; + if (!pubSocket.sendMore(INTERNAL_SERVICE_NAMES) || !msg.send(pubSocket)) { + LOGGER.atWarn().addArgument(msg.getServiceName()).log("could not notify service change for '{}'"); + } + break; + case W_HEARTBEAT: + if (workerReady) { + worker.updateExpiryTimeStamp(); + } else { + deleteWorker(worker, true); + } + break; + case DISCONNECT: + deleteWorker(worker, false); + break; + case PARTIAL: + case FINAL: + if (workerReady) { + final Client client = clients.get(msg.getServiceName()); + if (client == null || client.socket == null) { + break; + } + // need to replace clientID with service name + final byte[] serviceID = worker.service.nameBytes; + msg.senderID = msg.serviceNameBytes; + msg.protocol = PROT_CLIENT; + msg.serviceNameBytes = serviceID; + msg.send(client.socket); + workerWaiting(worker); + } else { + deleteWorker(worker, true); + } + break; + case W_NOTIFY: + // need to replace clientID with service name + final byte[] serviceID = worker.service.nameBytes; + msg.senderID = msg.serviceNameBytes; + msg.serviceNameBytes = serviceID; + msg.protocol = PROT_CLIENT; + msg.command = FINAL; + + dispatchMessageToMatchingSubscriber(msg); + + break; + default: + // N.B. not too verbose logging since we do not want that sloppy clients + // can bring down the broker through warning or info messages + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(msg).log("Majordomo broker invalid message: '{}'"); + } + break; + } + } + + private void dispatchMessageToMatchingSubscriber(final MdpMessage msg) { + // final String queryString = msg.topic.getQuery() + // final String replyService = msg.topic.getPath() + (queryString == null || queryString.isBlank() ? "" : ("?" + queryString)) + // N.B. for the time being only the path is matched - TODO: upgrade to full topic matching + for (String specificTopic : activeSubscriptions.keySet()) { + URI subTopic = URI.create(specificTopic); + if (!subTopic.getPath().startsWith(msg.topic.getPath())) { + continue; + } + pubSocket.sendMore(specificTopic); + msg.send(pubSocket); + } + + // publish also via router socket directly to known and previously subscribed clients + final List tClients = routerBasedSubscriptions.get(msg.topic.toString()); + if (tClients == null) { + return; + } + for (final byte[] clientID : tClients) { + msg.senderID = clientID; + msg.send(routerSocket); + } + } + + /** + * Look for & kill expired clients. + */ + protected void purgeClients() { + if (CLIENT_TIMEOUT <= 0) { + return; + } + for (String clientName : clients.keySet()) { // NOSONAR NOPMD copy because + // we are going to remove keys + Client client = clients.get(clientName); + if (client == null || client.expiry < System.currentTimeMillis()) { + clients.remove(clientName); + if (LOGGER.isDebugEnabled()) { + LOGGER.atDebug().addArgument(client).log("Majordomo broker deleting expired client: '{}'"); + } + } + } + } + + /** + * Look for & kill expired workers. Workers are oldest to most recent, so + * we stop at the first alive worker. + */ + protected void purgeWorkers() { + for (Worker w = waiting.peekFirst(); w != null && w.expiry < System.currentTimeMillis(); w = waiting.peekFirst()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.atInfo().addArgument(w.addressHex).addArgument(w.service == null ? "(unknown)" : w.service.name).log("Majordomo broker deleting expired worker: '{}' - service: '{}'"); + } + deleteWorker(waiting.pollFirst(), false); + } + } + + /** + * Look for & kill expired workers. Workers are oldest to most recent, so + * we stop at the first alive worker. + */ + protected void purgeDnsServices() { + if (System.currentTimeMillis() >= dnsHeartbeatAt) { + List cachedList = new ArrayList<>(dnsCache.values()); + final MdpMessage challengeMessage = new MdpMessage(null, PROT_CLIENT, W_HEARTBEAT, null, "dnsChallenge".getBytes(UTF_8), EMPTY_URI, EMPTY_FRAME, "", RBAC); + for (DnsServiceItem registeredService : cachedList) { + if (registeredService.serviceName.equalsIgnoreCase(brokerName)) { + registeredService.updateExpiryTimeStamp(); + } + // challenge remote broker with a HEARTBEAT + challengeMessage.senderID = registeredService.address; + challengeMessage.serviceNameBytes = registeredService.serviceName.getBytes(UTF_8); + challengeMessage.send(routerSocket); // NOPMD + if (System.currentTimeMillis() > registeredService.expiry) { + if (LOGGER.isInfoEnabled()) { + LOGGER.atInfo().addArgument(registeredService).log("Majordomo broker deleting expired dns service: '{}'"); + } + dnsCache.remove(registeredService.serviceName); + } + } + dnsHeartbeatAt = System.currentTimeMillis() + DNS_TIMEOUT; + } + } + + protected void registerDefaultServices(final RbacRole[] rbacRoles) { + // add simple internal Majordomo worker + final int nServiceThreads = 3; + + addInternalService(new MmiService(this, rbacRoles)); + addInternalService(new MmiOpenApi(this, rbacRoles)); + addInternalService(new MmiDns(this, rbacRoles)); + for (int i = 0; i < nServiceThreads; i++) { + addInternalService(new MmiEcho(this, rbacRoles)); // NOPMD valid instantiation inside loop + } + } + + /** + * Locates the service (creates if necessary). + * + * @param serviceName service name + * @param worker optional worker implementation (may be null) + * @return the existing (or new if absent) service this worker is responsible for + */ + protected Service requireService(final String serviceName, final BasicMdpWorker... worker) { + assert (serviceName != null); + final BasicMdpWorker w = worker.length > 0 ? worker[0] : null; + final Service service = services.computeIfAbsent(serviceName, s -> new Service(serviceName, serviceName.getBytes(UTF_8), w)); + if (w != null) { + w.start(); + } + return service; + } + + /** + * Finds the worker (creates if necessary). + * + * @param socket the socket + * @param address the address + * @param addressHex the address hex + * @param serviceName the service name + * @return the worker + */ + protected @NotNull Worker requireWorker(final Socket socket, final byte[] address, final String addressHex, final byte[] serviceName) { + assert (addressHex != null); + return workers.computeIfAbsent(addressHex, identity -> { + if (LOGGER.isInfoEnabled()) { + LOGGER.atInfo().addArgument(addressHex).log("registering new worker: '{}'"); + } + return new Worker(socket, address, addressHex, serviceName); + }); + } + + /** + * Send heartbeats to idle workers if it's time + */ + protected void sendHeartbeats() { + // Send heartbeats to idle workers if it's time + if (System.currentTimeMillis() >= heartbeatAt) { + final MdpMessage heartbeatMsg = new MdpMessage(null, PROT_WORKER, W_HEARTBEAT, null, EMPTY_FRAME, EMPTY_URI, EMPTY_FRAME, "", RBAC); + for (Worker worker : waiting) { + heartbeatMsg.senderID = worker.address; + heartbeatMsg.serviceNameBytes = worker.service.nameBytes; + heartbeatMsg.send(worker.socket); + } + heartbeatAt = System.currentTimeMillis() + HEARTBEAT_INTERVAL; + } + } + + /** + * Send heartbeats to the DNS server if necessary + * + * @param force sending regardless of time-out + */ + protected void sendDnsHeartbeats(boolean force) { + // Send heartbeats to idle workers if it's time + if (System.currentTimeMillis() >= dnsHeartbeatAt || force) { + final MdpMessage readyMsg = new MdpMessage(null, PROT_CLIENT, READY, brokerName.getBytes(UTF_8), "clientID".getBytes(UTF_8), URI.create(""), EMPTY_FRAME, "", RBAC); + for (String routerAddress : this.getRouterSockets()) { + readyMsg.topic = URI.create(routerAddress); + if (!dnsAddress.isBlank()) { + readyMsg.send(dnsSocket); // register with external DNS + } + // register with internal DNS + DnsServiceItem ret = dnsCache.computeIfAbsent(brokerName, s -> new DnsServiceItem(dnsSocket.getIdentity(), brokerName)); // NOPMD instantiation in loop necessary + ret.uri.add(URI.create(routerAddress)); + ret.updateExpiryTimeStamp(); + } + } + } + + /** + * This worker is now waiting for work. + * + * @param worker the worker + */ + protected void workerWaiting(Worker worker) { + // Queue to broker and service waiting lists + waiting.addLast(worker); + // TODO: evaluate addLast vs. push (addFirst) - latter should be more + // beneficial w.r.t. CPU context switches (reuses the same thread/context + // frequently + // do not know why original implementation wanted to spread across different + // workers (load balancing across different machines perhaps?!=) + // worker.service.waiting.addLast(worker); + worker.service.waiting.push(worker); + worker.updateExpiryTimeStamp(); + dispatch(worker.service); + } + + /** + * This defines a client service. + */ + protected static class Client { + protected final Socket socket; // Socket client is connected to + protected final String name; // client name + protected final byte[] nameBytes; // client name as byte array + protected final String nameHex; // client name as hex String + private final Deque requests = new ArrayDeque<>(); // List of client requests + protected long expiry = System.currentTimeMillis() + CLIENT_TIMEOUT; // Expires at unless heartbeat + + private Client(final Socket socket, final String name, final byte[] nameBytes) { + this.socket = socket; + this.name = name; + this.nameBytes = nameBytes == null ? name.getBytes(UTF_8) : nameBytes; + this.nameHex = strhex(nameBytes); + } + + private void offerToQueue(final MdpMessage msg) { + expiry = System.currentTimeMillis() + CLIENT_TIMEOUT; + requests.offer(msg); + } + + private MdpMessage pop() { + return requests.isEmpty() ? null : requests.poll(); + } + } + + /** + * This defines one worker, idle or active. + */ + protected static class Worker { + protected final Socket socket; // Socket worker is connected to + protected final byte[] address; // Address ID frame to route to + protected final String addressHex; // Address ID frame of worker expressed as hex-String + protected final byte[] serviceName; // service name of worker + + protected Service service; // Owning service, if known + protected long expiry; // Expires at unless heartbeat + + private Worker(final Socket socket, final byte[] address, final String addressHex, final byte[] serviceName) { // NOPMD direct storage of address OK + this.socket = socket; + this.address = address; + this.addressHex = addressHex; + this.serviceName = serviceName; + updateExpiryTimeStamp(); + } + + private void updateExpiryTimeStamp() { + expiry = System.currentTimeMillis() + HEARTBEAT_EXPIRY; + } + } + + /** + * This defines one DNS service item, idle or active. + */ + @SuppressWarnings("PMD.CommentDefaultAccessModifier") // needed for utility classes in the same package + static class DnsServiceItem { + protected final byte[] address; // Address ID frame to route to + protected final String serviceName; + protected final List uri = new NoDuplicatesList<>(); + protected long expiry; // Expires at unless heartbeat + + private DnsServiceItem(final byte[] address, final String serviceName) { // NOPMD direct storage of address OK + this.address = address; + this.serviceName = serviceName; + updateExpiryTimeStamp(); + } + + private void updateExpiryTimeStamp() { + expiry = System.currentTimeMillis() + DNS_TIMEOUT * HEARTBEAT_LIVENESS; + } + + @Override + public String toString() { + final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.UK); + return "DnsServiceItem{address=" + ZData.toString(address) + ", serviceName='" + serviceName + "', uri= '" + uri + "',expiry=" + expiry + " - " + sdf.format(expiry) + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (!(o instanceof DnsServiceItem)) { + return false; + } + DnsServiceItem that = (DnsServiceItem) o; + return serviceName.equals(that.serviceName); + } + + public String getDnsEntry() { + return '[' + serviceName + ": " + uri.stream().map(URI::toString).collect(Collectors.joining(",")) + ']'; + } + + public String getDnsEntryHtml() { + Optional webHandler = uri.stream().filter(u -> "https".equalsIgnoreCase(u.getScheme())).findFirst(); + if (webHandler.isEmpty()) { + webHandler = uri.stream().filter(u -> "http".equalsIgnoreCase(u.getScheme())).findFirst(); + } + final String wrappedService = webHandler.isEmpty() ? serviceName : wrapInAnchor(serviceName, webHandler.get()); + return '[' + wrappedService + ": " + uri.stream().map(u -> wrapInAnchor(u.toString(), u)).collect(Collectors.joining(",")) + "]"; + } + + @Override + public int hashCode() { + return serviceName.hashCode(); + } + } + + /** + * This defines a single service. + */ + protected class Service { + protected final String name; // Service name + protected final byte[] nameBytes; // Service name as byte array + protected final List mdpWorker = new ArrayList<>(); + protected final Map, Queue> requests = new HashMap<>(); // RBAC-based queuing + protected final Deque waiting = new ArrayDeque<>(); // List of waiting workers + protected final List internalWorkers = new ArrayList<>(); + protected byte[] serviceDescription; // service OpenAPI description + + private Service(final String name, final byte[] nameBytes, final BasicMdpWorker mdpWorker) { + this.name = name; + this.nameBytes = nameBytes == null ? name.getBytes(UTF_8) : nameBytes; + if (mdpWorker != null) { + this.mdpWorker.add(mdpWorker); + } + rbacRoles.forEach(role -> requests.put(role, new ArrayDeque<>())); + requests.put(BasicRbacRole.NULL, new ArrayDeque<>()); // add default queue + } + + private boolean requestsPending() { + return requests.entrySet().stream().anyMatch( + map -> !map.getValue().isEmpty()); + } + + private MdpMessage getNextPrioritisedMessage() { + for (RbacRole role : rbacRoles) { + final Queue queue = requests.get(role); // matched non-empty queue + if (!queue.isEmpty()) { + return queue.poll(); + } + } + final Queue queue = requests.get(BasicRbacRole.NULL); // default queue + return queue.isEmpty() ? null : queue.poll(); + } + + private void putPrioritisedMessage(final MdpMessage queuedMessage) { + if (queuedMessage.hasRbackToken()) { + // find proper RBAC queue + final RbacToken rbacToken = RbacToken.from(queuedMessage.rbacToken); + final Queue roleBasedQueue = requests.get(rbacToken.getRole()); + if (roleBasedQueue != null) { + roleBasedQueue.offer(queuedMessage); + } + } else { + requests.get(BasicRbacRole.NULL).offer(queuedMessage); + } + } + } + + protected static String getLocalHostName() { + String ip; + try (DatagramSocket socket = new DatagramSocket()) { + socket.connect(InetAddress.getByName("8.8.8.8"), 10_002); // NOPMD - bogus hardcoded IP acceptable in this context + if (socket.getLocalAddress() == null) { + throw new UnknownHostException("bogus exception can be ignored"); + } + ip = socket.getLocalAddress().getHostAddress(); + + if (ip != null) { + return ip; + } + } catch (final SocketException | UnknownHostException e) { + LOGGER.atError().setCause(e).log("getLocalHostName()"); + } + return "localhost"; + } +} diff --git a/server/src/main/java/io/opencmw/server/MajordomoWorker.java b/server/src/main/java/io/opencmw/server/MajordomoWorker.java new file mode 100644 index 00000000..561c8c49 --- /dev/null +++ b/server/src/main/java/io/opencmw/server/MajordomoWorker.java @@ -0,0 +1,203 @@ +package io.opencmw.server; + +import static io.opencmw.OpenCmwProtocol.Command.W_NOTIFY; +import static io.opencmw.OpenCmwProtocol.EMPTY_FRAME; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; + +import java.net.URI; +import java.util.Arrays; + +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.ZContext; + +import io.opencmw.MimeType; +import io.opencmw.OpenCmwProtocol; +import io.opencmw.OpenCmwProtocol.MdpMessage; +import io.opencmw.QueryParameterParser; +import io.opencmw.rbac.RbacRole; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.JsonSerialiser; + +/** + * MajordomoWorker implementation including de-serialising and re-serialising to domain-objects. + *

+ * This implements GET/SET/NOTIFY handlers that are driven by PoJo domain objects. + * + * @author rstein + * + * @param generic type for the query/context mapping object + * @param generic type for the input domain object + * @param generic type for the output domain object (also notify) + */ +@SuppressWarnings("PMD.DataClass") // PMD - false positive data class +@MetaInfo(description = "default MajordomoWorker implementation") +public class MajordomoWorker extends BasicMdpWorker { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoWorker.class); + private static final int MAX_BUFFER_SIZE = 4000; + protected final IoBuffer defaultBuffer = new FastByteBuffer(MAX_BUFFER_SIZE, true, null); + protected final IoBuffer defaultNotifyBuffer = new FastByteBuffer(MAX_BUFFER_SIZE, true, null); + protected final IoClassSerialiser deserialiser = new IoClassSerialiser(defaultBuffer); + protected final IoClassSerialiser serialiser = new IoClassSerialiser(defaultBuffer); + protected final IoClassSerialiser notifySerialiser = new IoClassSerialiser(defaultNotifyBuffer); + protected final Class contextClassType; + protected final Class inputClassType; + protected final Class outputClassType; + protected Handler handler; + protected Handler htmlHandler; + + public MajordomoWorker(final String brokerAddress, final String serviceName, + @NotNull final Class contextClassType, + @NotNull final Class inputClassType, + @NotNull final Class outputClassType, final RbacRole... rbacRoles) { + this(null, brokerAddress, serviceName, contextClassType, inputClassType, outputClassType, rbacRoles); + } + + public MajordomoWorker(final ZContext ctx, final String serviceName, + @NotNull final Class contextClassType, + @NotNull final Class inputClassType, + @NotNull final Class outputClassType, final RbacRole... rbacRoles) { + this(ctx, "inproc://broker", serviceName, contextClassType, inputClassType, outputClassType, rbacRoles); + } + + protected MajordomoWorker(final ZContext ctx, final String brokerAddress, final String serviceName, + @NotNull final Class contextClassType, + @NotNull final Class inputClassType, + @NotNull final Class outputClassType, final RbacRole... rbacRoles) { + super(ctx, brokerAddress, serviceName, rbacRoles); + + this.contextClassType = contextClassType; + this.inputClassType = inputClassType; + this.outputClassType = outputClassType; + deserialiser.setAutoMatchSerialiser(false); + serialiser.setAutoMatchSerialiser(false); + notifySerialiser.setAutoMatchSerialiser(false); + serialiser.setMatchedIoSerialiser(BinarySerialiser.class); + notifySerialiser.setMatchedIoSerialiser(BinarySerialiser.class); + + try { + // check if velocity is available + Class.forName("org.apache.velocity.app.VelocityEngine"); + setHtmlHandler(new DefaultHtmlHandler<>(this.getClass(), null, null)); + } catch (ClassNotFoundException e) { + LOGGER.atInfo().addArgument("velocity engine not present - omitting setting DefaultHtmlHandler()"); + } + + super.registerHandler(c -> { + final URI reqTopic = c.req.topic; + final String queryString = reqTopic.getQuery(); + final C requestCtx = QueryParameterParser.parseQueryParameter(contextClassType, c.req.topic.getQuery()); + final C replyCtx = QueryParameterParser.parseQueryParameter(contextClassType, c.req.topic.getQuery()); // reply is initially a copy of request + final MimeType requestedMimeType = QueryParameterParser.getMimeType(queryString); + + final I input; + if (c.req.data.length > 0) { + switch (requestedMimeType) { + case HTML: + case JSON: + case JSON_LD: + deserialiser.setDataBuffer(FastByteBuffer.wrap(c.req.data)); + deserialiser.setMatchedIoSerialiser(JsonSerialiser.class); + input = deserialiser.deserialiseObject(inputClassType); + break; + case CMWLIGHT: + deserialiser.setDataBuffer(FastByteBuffer.wrap(c.req.data)); + deserialiser.setMatchedIoSerialiser(CmwLightSerialiser.class); + input = deserialiser.deserialiseObject(inputClassType); + break; + case BINARY: + case UNKNOWN: + default: + deserialiser.setDataBuffer(FastByteBuffer.wrap(c.req.data)); + deserialiser.setMatchedIoSerialiser(BinarySerialiser.class); + input = deserialiser.deserialiseObject(inputClassType); + break; + } + } else { + // return default input object + input = inputClassType.getDeclaredConstructor().newInstance(); + } + + final O output = outputClassType.getDeclaredConstructor().newInstance(); + + // call user-handler + handler.handle(c, requestCtx, input, replyCtx, output); + + final String replyQuery = QueryParameterParser.generateQueryParameter(replyCtx); + c.rep.topic = new URI(reqTopic.getScheme(), reqTopic.getAuthority(), reqTopic.getPath(), replyQuery, reqTopic.getFragment()); + final MimeType replyMimeType = QueryParameterParser.getMimeType(replyQuery); + + defaultBuffer.reset(); + switch (replyMimeType) { + case HTML: + htmlHandler.handle(c, requestCtx, input, replyCtx, output); + break; + case JSON: + case JSON_LD: + serialiser.setMatchedIoSerialiser(JsonSerialiser.class); + serialiser.getMatchedIoSerialiser().setBuffer(defaultBuffer); + serialiser.serialiseObject(output); + defaultBuffer.flip(); + c.rep.data = Arrays.copyOf(defaultBuffer.elements(), defaultBuffer.limit() + 4); + break; + case CMWLIGHT: + serialiser.setMatchedIoSerialiser(CmwLightSerialiser.class); + serialiser.getMatchedIoSerialiser().setBuffer(defaultBuffer); + serialiser.serialiseObject(output); + defaultBuffer.flip(); + c.rep.data = Arrays.copyOf(defaultBuffer.elements(), defaultBuffer.limit()); + break; + case BINARY: + default: + serialiser.setMatchedIoSerialiser(BinarySerialiser.class); + serialiser.getMatchedIoSerialiser().setBuffer(defaultBuffer); + serialiser.serialiseObject(output); + defaultBuffer.flip(); + c.rep.data = Arrays.copyOf(defaultBuffer.elements(), defaultBuffer.limit()); + break; + } + }); + } + + @Override + public BasicMdpWorker registerHandler(final RequestHandler requestHandler) { + throw new UnsupportedOperationException("do not overwrite low-level request handler, use either 'setHandler(...)' or " + BasicMdpWorker.class.getName() + " directly"); + } + + public Handler getHandler() { + return handler; + } + + public void setHandler(final Handler handler) { + this.handler = handler; + } + + public Handler getHtmlHandler() { + return htmlHandler; + } + + public void setHtmlHandler(Handler htmlHandler) { + this.htmlHandler = htmlHandler; + } + + public void notify(final C replyCtx, final O reply) { + defaultNotifyBuffer.reset(); + notifySerialiser.serialiseObject(reply); + defaultNotifyBuffer.flip(); + final byte[] data = Arrays.copyOf(defaultNotifyBuffer.elements(), defaultNotifyBuffer.limit()); + URI topic = URI.create(serviceName + '?' + QueryParameterParser.generateQueryParameter(replyCtx)); + MdpMessage notifyMessage = new MdpMessage(null, PROT_WORKER, W_NOTIFY, serviceBytes, EMPTY_FRAME, topic, data, "", RBAC); + + super.notify(notifyMessage); + } + + public interface Handler { + void handle(OpenCmwProtocol.Context ctx, C requestCtx, I request, C replyCtx, O reply); + } +} diff --git a/server/src/main/java/io/opencmw/server/MmiServiceHelper.java b/server/src/main/java/io/opencmw/server/MmiServiceHelper.java new file mode 100644 index 00000000..e10c9495 --- /dev/null +++ b/server/src/main/java/io/opencmw/server/MmiServiceHelper.java @@ -0,0 +1,89 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.net.URI; +import java.util.stream.Collectors; + +import io.opencmw.MimeType; +import io.opencmw.QueryParameterParser; +import io.opencmw.rbac.RbacRole; +import io.opencmw.serialiser.annotations.MetaInfo; + +public final class MmiServiceHelper { + public static final String INTERNAL_SERVICE_NAMES = "mmi.service"; + public static final String INTERNAL_SERVICE_OPENAPI = "mmi.openapi"; + public static final String INTERNAL_SERVICE_DNS = "mmi.dns"; + public static final String INTERNAL_SERVICE_ECHO = "mmi.echo"; + + private MmiServiceHelper() { + // nothing to instantiate + } + + public static boolean isHtmlRequest(final URI topic) { + return topic != null && topic.getQuery() != null && MimeType.HTML == QueryParameterParser.getMimeType(topic.getQuery()); + } + + public static String wrapInAnchor(final String text, final URI uri) { + if (uri.getScheme() == null || uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + return "" + text + ""; + } + return text; + } + + @MetaInfo(description = "output = input : echo service is that complex :-)", unit = "MMI Echo Service") + public static class MmiEcho extends BasicMdpWorker { + public MmiEcho(final MajordomoBroker broker, final RbacRole... rbacRoles) { + super(broker.getContext(), INTERNAL_SERVICE_ECHO, rbacRoles); + this.registerHandler(ctx -> ctx.rep.data = ctx.req.data); // output = input : echo service is complex :-) + } + } + + @MetaInfo(description = "Dynamic Name Service (DNS) returning registered internal and external broker endpoints' URIs (protocol, host ip, port etc.)", unit = "MMI DNS Service") + public static class MmiDns extends BasicMdpWorker { + public MmiDns(final MajordomoBroker broker, final RbacRole... rbacRoles) { + super(broker.getContext(), INTERNAL_SERVICE_DNS, rbacRoles); + this.registerHandler(ctx -> { + if (isHtmlRequest(ctx.req.topic)) { + ctx.rep.data = broker.dnsCache.values().stream().map(MajordomoBroker.DnsServiceItem::getDnsEntryHtml).collect(Collectors.joining(",
")).getBytes(UTF_8); + } else { + ctx.rep.data = broker.dnsCache.values().stream().map(MajordomoBroker.DnsServiceItem::getDnsEntry).collect(Collectors.joining(",")).getBytes(UTF_8); + } + }); + } + } + + @MetaInfo(description = "endpoint returning OpenAPI definitions", unit = "MMI OpenAPI Worker Class Definitions") + public static class MmiOpenApi extends BasicMdpWorker { + public MmiOpenApi(final MajordomoBroker broker, final RbacRole... rbacRoles) { + super(broker.getContext(), INTERNAL_SERVICE_OPENAPI, rbacRoles); + this.registerHandler(context -> { + final String serviceName = context.req.data == null ? "" : new String(context.req.data, UTF_8); + final MajordomoBroker.Service service = broker.services.get(serviceName); + if (service == null) { + throw new IllegalArgumentException("requested invalid service name '" + serviceName + "' msg " + context.req); + } + context.rep.data = service.serviceDescription; + }); + } + } + + @MetaInfo(description = "definition according to http://rfc.zeromq.org/spec:8", unit = "MMI Service/Property Definitions") + public static class MmiService extends BasicMdpWorker { + public MmiService(final MajordomoBroker broker, final RbacRole... rbacRoles) { + super(broker.getContext(), INTERNAL_SERVICE_NAMES, rbacRoles); + this.registerHandler(context -> { + final String serviceName = (context.req.data == null) ? "" : new String(context.req.data, UTF_8); + if (serviceName.isBlank()) { + if (isHtmlRequest(context.req.topic)) { + context.rep.data = broker.services.keySet().stream().sorted().map(s -> wrapInAnchor(s, URI.create("/" + s))).collect(Collectors.joining(",")).getBytes(UTF_8); + } else { + context.rep.data = broker.services.keySet().stream().sorted().collect(Collectors.joining(",")).getBytes(UTF_8); + } + } else { + context.rep.data = (broker.services.containsKey(serviceName) ? "200" : "400").getBytes(UTF_8); + } + }); + } + } +} diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java new file mode 100644 index 00000000..d7c0951c --- /dev/null +++ b/server/src/main/java/module-info.java @@ -0,0 +1,12 @@ +open module io.opencmw.server { + requires org.slf4j; + requires io.opencmw.serialiser; + requires io.opencmw; + requires jeromq; + requires java.management; + requires org.apache.commons.lang3; + requires velocity.engine.core; + requires org.jetbrains.annotations; + + exports io.opencmw.server; +} \ No newline at end of file diff --git a/server/src/main/resources/velocity/property/defaultPropertyLayout.vm b/server/src/main/resources/velocity/property/defaultPropertyLayout.vm new file mode 100644 index 00000000..c087b7d3 --- /dev/null +++ b/server/src/main/resources/velocity/property/defaultPropertyLayout.vm @@ -0,0 +1,218 @@ + + +#if (!$noMenu) + + + +

+

Property: '$service'

+

$mdpMetaData.getFieldDescription()

+
+ + +
+Subscribe: + + +
+ +
+ +#else + +#macro( renderPropertyData $propertyDataName $propertyData ) + + + + + + #foreach ($field in $propertyData.keySet()) + #if ($field.getFieldUnit()) + #set($unit = "["+$field.getFieldUnit()+"]:") + #else + #set($unit = ":") + #end + #if ($field.isEnum()) + #set($unit = "["+$field.getTypeNameSimple()+"]:") + #end + #if ($field.getFieldDescription()) + #set($description = $field.getFieldDescription()) + #else + #set($description = "") + #end + #set($value = $propertyData.get($field)) + + + + + + #end + +
$propertyDataName Field NameUnit:Field Value:
$unit + #if ($field.isEnum()) + + #else + #if ($value == "true" || $value == "false") + #set($type = 'type="checkbox"') + #else + #set($type = 'type="text" placeholder=""') + #end + + #end +
+#end + + +
+
+ + +
+
+ #renderPropertyData( $requestCtx.getClass().getSimpleName() $requestCtxClassData ) + ## pushes form data as query string and reloads with the new URI +
+
+
+
+ + + +#if ($request && $requestClassData && $requestClassData.size() != 0 && $request.getClass().getSimpleName() != "NoData") +
+
+ + +
+
+ #renderPropertyData( $request.getClass().getSimpleName() $requestClassData ) + +
+
+
+
+#end + +
+
+ + +
+
+ #renderPropertyData( $replyCtx.getClass().getSimpleName() $replyCtxClassData ) +
+
+
+
+ +
+
+ + +
+
+ #renderPropertyData( $reply.getClass().getSimpleName() $replyClassData ) + #if ( $reply.getClass().getSimpleName() == "BinaryData") + + #end +
+
+
+
+#end \ No newline at end of file diff --git a/server/src/test/java/io/opencmw/server/BasicMdpWorkerTest.java b/server/src/test/java/io/opencmw/server/BasicMdpWorkerTest.java new file mode 100644 index 00000000..df1fb356 --- /dev/null +++ b/server/src/test/java/io/opencmw/server/BasicMdpWorkerTest.java @@ -0,0 +1,111 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.*; + +import static io.opencmw.OpenCmwProtocol.Command.W_NOTIFY; +import static io.opencmw.OpenCmwProtocol.EMPTY_FRAME; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; + +import java.net.URI; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.zeromq.ZContext; + +import io.opencmw.OpenCmwProtocol; +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.utils.SystemProperties; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class BasicMdpWorkerTest { + @BeforeAll + void init() { + System.getProperties().setProperty("OpenCMW.heartBeat", "50"); + assertEquals(50, SystemProperties.getValueIgnoreCase("OpenCMW.heartBeat", 2500), "reduced heart-beat interval"); + } + + @Test + void testConstructors() { + assertDoesNotThrow(() -> new BasicMdpWorker(new ZContext(), "testService")); + assertDoesNotThrow(() -> new BasicMdpWorker(new ZContext(), "testService", BasicRbacRole.ADMIN)); + assertDoesNotThrow(() -> new BasicMdpWorker("mdp://localhost", "testService")); + assertDoesNotThrow(() -> new BasicMdpWorker("mdp://localhost", "testService", BasicRbacRole.ADMIN)); + } + + @Test + void testGetterSetter() { + BasicMdpWorker worker = new BasicMdpWorker(new ZContext(), "testService", BasicRbacRole.ADMIN); + BasicMdpWorker worker2 = new BasicMdpWorker(new ZContext(), "testService", BasicRbacRole.ADMIN); + + assertThat(worker.getReconnectDelay(), not(equalTo(Duration.of(123, ChronoUnit.MILLIS)))); + assertDoesNotThrow(() -> worker.setReconnectDelay(123, TimeUnit.MILLISECONDS)); + assertThat(worker.getReconnectDelay(), equalTo(Duration.of(123, ChronoUnit.MILLIS))); + + assertEquals("testService", worker.getServiceName()); + + assertNotNull(worker.getUniqueID()); + assertNotNull(worker2.getUniqueID()); + assertThat(worker.getUniqueID(), not(equalTo(worker2.getUniqueID()))); + + assertNotNull(worker.getRbacRoles()); + assertThat(worker.getRbacRoles(), contains(BasicRbacRole.ADMIN)); + + final BasicMdpWorker.RequestHandler handler = c -> {}; + assertThat(worker.getRequestHandler(), not(equalTo(handler))); + assertDoesNotThrow(() -> worker.registerHandler(handler)); + assertThat(worker.getRequestHandler(), equalTo(handler)); + } + + @ParameterizedTest + @ValueSource(booleans = { true /*, false */ }) + void testBasicThreadHander(boolean internal) { + // N.B. this is not a full-blown tests and only covers the basic failure mode, full tests is implemented in MajordomoBrokerTests + BasicMdpWorker worker = internal ? new BasicMdpWorker(new ZContext(), "testService", BasicRbacRole.ADMIN) : new BasicMdpWorker("mdp://*:8080/", "testService", BasicRbacRole.ADMIN); + + final AtomicBoolean run = new AtomicBoolean(false); + final AtomicBoolean stop = new AtomicBoolean(false); + new Thread(() -> { + run.set(true); + worker.start(); + stop.set(true); + }).start(); + + await().alias("wait for thread to start worker").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(true)); + await().alias("wait for thread to have finished starting worker").atMost(1, TimeUnit.SECONDS).until(stop::get, equalTo(true)); + run.set(false); + stop.set(false); + + assertTrue(worker.runSocketHandlerLoop.get(), "run loop is running"); + + // check basic notification + final OpenCmwProtocol.MdpMessage msg = new OpenCmwProtocol.MdpMessage(null, PROT_WORKER, W_NOTIFY, "testService".getBytes(UTF_8), EMPTY_FRAME, URI.create(""), EMPTY_FRAME, "", null); + assertFalse(worker.notify(msg)); // is filtered: no subscription -> false + assertTrue(worker.notifyRaw(msg)); // is unfiltered: no subscription -> true + + // wait for five heartbeats -> checks poller, heartbeat and reconnect features (to some extend) + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(5L * BasicMdpWorker.HEARTBEAT_INTERVAL)); + + new Thread(() -> { + run.set(true); + worker.stopWorker(); + stop.set(true); + }).start(); + await().alias("wait for thread to stop worker").atMost(1, TimeUnit.SECONDS).until(run::get, equalTo(true)); + await().alias("wait for thread to have finished stopping worker").atMost(1, TimeUnit.SECONDS).until(stop::get, equalTo(true)); + } +} \ No newline at end of file diff --git a/server/src/test/java/io/opencmw/server/MajordomoBrokerTests.java b/server/src/test/java/io/opencmw/server/MajordomoBrokerTests.java new file mode 100644 index 00000000..f35c72cb --- /dev/null +++ b/server/src/test/java/io/opencmw/server/MajordomoBrokerTests.java @@ -0,0 +1,368 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.*; + +import static io.opencmw.OpenCmwProtocol.Command.FINAL; +import static io.opencmw.OpenCmwProtocol.Command.GET_REQUEST; +import static io.opencmw.OpenCmwProtocol.MdpMessage; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_CLIENT; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_WORKER; +import static io.opencmw.utils.AnsiDefs.ANSI_RED; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +import org.junit.jupiter.api.Test; +import org.zeromq.SocketType; +import org.zeromq.Utils; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.rbac.RbacToken; + +class MajordomoBrokerTests { + private static final byte[] DEFAULT_RBAC_TOKEN = new RbacToken(BasicRbacRole.ADMIN, "HASHCODE").getBytes(); + private static final String DEFAULT_MMI_SERVICE = "mmi.service"; + private static final String DEFAULT_ECHO_SERVICE = "mmi.echo"; + private static final String DEFAULT_REQUEST_MESSAGE = "Hello World!"; + private static final byte[] DEFAULT_REQUEST_MESSAGE_BYTES = DEFAULT_REQUEST_MESSAGE.getBytes(UTF_8); + + /** + * Main method - create and start new broker. + * + * @param args none + */ + public static void main(String[] args) { + MajordomoBroker broker = new MajordomoBroker("TestMdpBroker", "tcp://*:5555", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that + // controls threads Can be called multiple times with different endpoints + broker.bind("tcp://*:5555"); + broker.bind("tcp://*:5556"); + + for (int i = 0; i < 10; i++) { + // simple internalSock echo + BasicMdpWorker workerSession = new BasicMdpWorker(broker.getContext(), "inproc.echo", BasicRbacRole.ADMIN); // NOPMD safe instantiation + workerSession.registerHandler(ctx -> ctx.rep.data = ctx.req.data); // output = input : echo service is complex :-) + workerSession.start(); + } + + broker.start(); + } + + @Test + void basicLowLevelRequestReplyTest() throws IOException { + MajordomoBroker broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final String brokerAddress = broker.bind("mdp://*:" + Utils.findOpenPort()); + assertFalse(broker.isRunning(), "broker not running"); + broker.start(); + assertTrue(broker.isRunning(), "broker running"); + // test interfaces + assertNotNull(broker.getContext()); + assertNotNull(broker.getInternalRouterSocket()); + assertNotNull(broker.getServices()); + assertEquals(4, broker.getServices().size()); + assertDoesNotThrow(() -> broker.addInternalService(new BasicMdpWorker(broker.getContext(), "demoService"))); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); // wait until services are started + assertEquals(5, broker.getServices().size()); + assertDoesNotThrow(() -> broker.removeService("demoService")); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); // wait until services are stopped + assertEquals(5, broker.getServices().size()); + + // wait until all services are initialised + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(200)); + + final ZMQ.Socket clientSocket = broker.getContext().createSocket(SocketType.DEALER); + clientSocket.setIdentity("demoClient".getBytes(UTF_8)); + clientSocket.connect(brokerAddress.replace("mdp", "tcp")); + + // wait until client is connected + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(200)); + + final byte[] clientRequestID = "unit-test-clientRequestID".getBytes(UTF_8); + new MdpMessage(null, PROT_CLIENT, GET_REQUEST, DEFAULT_ECHO_SERVICE.getBytes(UTF_8), clientRequestID, URI.create(DEFAULT_ECHO_SERVICE), DEFAULT_REQUEST_MESSAGE_BYTES, "", new byte[0]).send(clientSocket); + final MdpMessage clientMessage = MdpMessage.receive(clientSocket); + assertNotNull(clientMessage, "reply message w/o RBAC token not being null"); + assertNotNull(clientMessage.toString()); + assertNotNull(clientMessage.senderID); // default dealer socket does not export sender ID (only ROUTER and/or enabled sockets) + assertEquals(MdpSubProtocol.PROT_CLIENT, clientMessage.protocol, "equal protocol"); + assertEquals(FINAL, clientMessage.command, "matching command"); + assertArrayEquals(DEFAULT_ECHO_SERVICE.getBytes(UTF_8), clientMessage.serviceNameBytes, "equal service name"); + assertNotNull(clientMessage.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, clientMessage.data, "equal data"); + assertFalse(clientMessage.hasRbackToken()); + assertNotNull(clientMessage.rbacToken); + assertEquals(0, clientMessage.rbacToken.length, "rback token length (should be 0: not defined)"); + + broker.stopBroker(); + } + + @Test + void basicSynchronousRequestReplyTest() throws IOException { + final MajordomoBroker broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final String brokerAddress = broker.bind("mdp://*:" + Utils.findOpenPort()); + broker.start(); + assertEquals(4, broker.getServices().size()); + + // add external (albeit inproc) Majordomo worker to the broker + BasicMdpWorker internal = new BasicMdpWorker(broker.getContext(), "inproc.echo", BasicRbacRole.ADMIN); + internal.registerHandler(ctx -> ctx.rep.data = ctx.req.data); // output = input : echo service is complex :-) + internal.start(); + + // add external Majordomo worker to the broker + BasicMdpWorker external = new BasicMdpWorker(broker.getContext(), "ext.echo", BasicRbacRole.ADMIN); + external.registerHandler(ctx -> ctx.rep.data = ctx.req.data); // output = input : echo service is complex :-) + external.start(); + + // add external (albeit inproc) Majordomo worker to the broker + BasicMdpWorker exceptionService = new BasicMdpWorker(broker.getContext(), "inproc.exception", BasicRbacRole.ADMIN); + exceptionService.registerHandler(input -> { throw new IllegalAccessError("this is always thrown"); }); // always throw an exception + exceptionService.start(); + + // wait until all services are initialised + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(200)); + assertEquals(7, broker.getServices().size()); + + // using simple synchronous client + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + assertEquals(3, clientSession.getRetries()); + assertDoesNotThrow(() -> clientSession.setRetries(4)); + assertEquals(4, clientSession.getRetries()); + assertEquals(2500, clientSession.getTimeout()); + assertDoesNotThrow(() -> clientSession.setTimeout(2000)); + assertEquals(2000, clientSession.getTimeout()); + assertNotNull(clientSession.getUniqueID()); + + { + final String serviceName = "mmi.echo"; + final MdpMessage replyWithoutRbac = clientSession.send(serviceName, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.data, "equal data"); + } + + { + final String serviceName = "inproc.echo"; + final MdpMessage replyWithoutRbac = clientSession.send(serviceName, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.data, "equal data"); + } + + { + final String serviceName = "ext.echo"; + final MdpMessage replyWithoutRbac = clientSession.send(serviceName, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.data, "equal data"); + } + + { + final String serviceName = "inproc.exception"; + final MdpMessage replyWithoutRbac = clientSession.send(serviceName, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertNotNull(replyWithoutRbac.errors, "user-data not being null"); + assertFalse(replyWithoutRbac.errors.isBlank(), "check that error stack trace is non-null/non-blank"); + final String refString = ANSI_RED + "io.opencmw.server.BasicMdpWorker caught exception for service 'inproc.exception'"; + assertEquals(refString, replyWithoutRbac.errors.substring(0, refString.length()), "correct exception message"); + } + + { + final String serviceName = "mmi.echo"; + final MdpMessage replyWithRbac = clientSession.send(serviceName, DEFAULT_REQUEST_MESSAGE_BYTES, DEFAULT_RBAC_TOKEN); // with RBAC + assertNotNull(replyWithRbac, "reply message with RBAC token not being null"); + assertNotNull(replyWithRbac.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithRbac.data, "equal data"); + assertNotNull(replyWithRbac.rbacToken, "RBAC token not being null"); + assertEquals(0, replyWithRbac.rbacToken.length, "non-defined RBAC token length"); + } + + internal.stopWorker(); + external.stopWorker(); + exceptionService.stopWorker(); + broker.stopBroker(); + } + + @Test + void basicMmiTests() throws IOException { + MajordomoBroker broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final int openPort = Utils.findOpenPort(); + broker.bind("tcp://*:" + openPort); + broker.start(); + + // using simple synchronous client + MajordomoTestClientSync clientSession = new MajordomoTestClientSync("tcp://localhost:" + openPort, "customClientName"); + + { + final MdpMessage replyWithoutRbac = clientSession.send("mmi.echo", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, replyWithoutRbac.data, "MMI echo service request"); + } + + { + final MdpMessage replyWithoutRbac = clientSession.send(DEFAULT_MMI_SERVICE, DEFAULT_MMI_SERVICE.getBytes(UTF_8)); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertEquals("200", new String(replyWithoutRbac.data, UTF_8), "known MMI service request"); + } + + { + final MdpMessage replyWithoutRbac = clientSession.send(DEFAULT_MMI_SERVICE, DEFAULT_ECHO_SERVICE.getBytes(UTF_8)); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertEquals("200", new String(replyWithoutRbac.data, UTF_8), "known MMI service request"); + } + + { + // MMI service request: service should not exist + final MdpMessage replyWithoutRbac = clientSession.send(DEFAULT_MMI_SERVICE, DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertEquals("400", new String(replyWithoutRbac.data, UTF_8), "known MMI service request"); + } + + { + // unknown service name + final MdpMessage replyWithoutRbac = clientSession.send("unknownService", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(replyWithoutRbac, "reply message w/o RBAC token not being null"); + assertNotNull(replyWithoutRbac.data, "user-data not being null"); + assertEquals("501", new String(replyWithoutRbac.data, UTF_8), "unknown MMI service request"); + } + + broker.stopBroker(); + } + + @Test + void basicASynchronousRequestReplyTest() throws IOException { + MajordomoBroker broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final int openPort = Utils.findOpenPort(); + broker.bind("tcp://*:" + openPort); + broker.start(); + + final AtomicInteger counter = new AtomicInteger(0); + new Thread(() -> { + // using simple synchronous client + MajordomoTestClientAsync clientSession = new MajordomoTestClientAsync("tcp://localhost:" + openPort); + assertEquals(2500, clientSession.getTimeout()); + assertDoesNotThrow(() -> clientSession.setTimeout(2000)); + assertEquals(2000, clientSession.getTimeout()); + + // send bursts of 10 messages + for (int i = 0; i < 5; i++) { + clientSession.send("mmi.echo", DEFAULT_REQUEST_MESSAGE_BYTES); + clientSession.send(DEFAULT_ECHO_SERVICE, DEFAULT_REQUEST_MESSAGE_BYTES); + } + + // receive bursts of 10 messages + for (int i = 0; i < 10; i++) { + final MdpMessage reply = clientSession.recv(); + assertNotNull(reply, "reply message w/o RBAC token not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, reply.data); + counter.getAndIncrement(); + } + }).start(); + + await().alias("wait for reply messages").atMost(1, TimeUnit.SECONDS).until(counter::get, equalTo(10)); + assertEquals(10, counter.get(), "received expected number of replies"); + + broker.stopBroker(); + } + + @Test + void testSubscription() throws IOException { + final MajordomoBroker broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final String brokerAddress = broker.bind("mdp://*:" + Utils.findOpenPort()); + final String brokerPubAddress = broker.bind("mds://*:" + Utils.findOpenPort()); + broker.start(); + + final String testServiceName = "device/property"; + final byte[] testServiceBytes = "device/property".getBytes(UTF_8); + + // add external (albeit inproc) Majordomo worker to the broker + BasicMdpWorker internal = new BasicMdpWorker(broker.getContext(), testServiceName, BasicRbacRole.ADMIN); + internal.registerHandler(ctx -> ctx.rep.data = ctx.req.data); // output = input : echo service is complex :-) + internal.start(); + + final MdpMessage testMessage = new MdpMessage(null, PROT_WORKER, FINAL, testServiceBytes, "clientRequestID".getBytes(UTF_8), URI.create(new String(testServiceBytes)), DEFAULT_REQUEST_MESSAGE_BYTES, "", new byte[0]); + + final AtomicInteger counter = new AtomicInteger(0); + final AtomicBoolean run = new AtomicBoolean(true); + final AtomicBoolean started1 = new AtomicBoolean(false); + new Thread(() -> { + // using simple synchronous client + MajordomoTestClientAsync clientSession = new MajordomoTestClientAsync(brokerAddress); + assertEquals(2500, clientSession.getTimeout()); + assertDoesNotThrow(() -> clientSession.setTimeout(2000)); + assertEquals(2000, clientSession.getTimeout()); + clientSession.subscribe(testServiceBytes); + + // send bursts of 10 messages + for (int i = 0; i < 10 && run.get(); i++) { + started1.set(true); + final MdpMessage reply = clientSession.recv(); + assertNotNull(reply, "reply message w/o RBAC token not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, reply.data); + counter.getAndIncrement(); + } + }).start(); + + // low-level subscription + final AtomicInteger subCounter = new AtomicInteger(0); + final AtomicBoolean started2 = new AtomicBoolean(false); + final Thread subcriptionThread = new Thread(() -> { + // low-level subscription + final ZMQ.Socket sub = broker.getContext().createSocket(SocketType.SUB); + sub.setHWM(0); + sub.connect(brokerPubAddress.replace("mds://", "tcp://")); + sub.subscribe("device/property"); + sub.subscribe("device/otherProperty"); + sub.unsubscribe("device/otherProperty"); + while (run.get() && !Thread.interrupted()) { + started2.set(true); + final ZMsg msg = ZMsg.recvMsg(sub, false); + if (msg == null) { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + continue; + } + subCounter.getAndIncrement(); + } + sub.unsubscribe("device/property"); + }); + subcriptionThread.start(); + + // wait until all services are initialised + await().alias("wait for thread1 to start").atMost(1, TimeUnit.SECONDS).until(started1::get, equalTo(true)); + await().alias("wait for thread2 to start").atMost(1, TimeUnit.SECONDS).until(started2::get, equalTo(true)); + // send bursts of 10 messages + for (int i = 0; i < 10; i++) { + internal.notify(testMessage); + } + + await().alias("wait for reply messages").atMost(2, TimeUnit.SECONDS).until(counter::get, equalTo(10)); + run.set(false); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); + assertFalse(subcriptionThread.isAlive(), "subscription thread shut-down"); + assertEquals(10, counter.get(), "received expected number of replies"); + assertEquals(10, subCounter.get(), "received expected number of subscription replies"); + + broker.stopBroker(); + } +} diff --git a/server/src/test/java/io/opencmw/server/MajordomoTestClientAsync.java b/server/src/test/java/io/opencmw/server/MajordomoTestClientAsync.java new file mode 100644 index 00000000..1f2c1750 --- /dev/null +++ b/server/src/test/java/io/opencmw/server/MajordomoTestClientAsync.java @@ -0,0 +1,120 @@ +package io.opencmw.server; + +import static io.opencmw.OpenCmwProtocol.Command.GET_REQUEST; +import static io.opencmw.OpenCmwProtocol.Command.SUBSCRIBE; +import static io.opencmw.OpenCmwProtocol.EMPTY_FRAME; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_CLIENT; + +import java.net.URI; +import java.nio.charset.StandardCharsets; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +import io.opencmw.OpenCmwProtocol; + +/** + * Majordomo Protocol Client API, asynchronous Java version. Implements the + * OpenCmwProtocol/Worker spec at http://rfc.zeromq.org/spec:7. + */ +public class MajordomoTestClientAsync { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoTestClientAsync.class); + private final String broker; + private final ZContext ctx; + private ZMQ.Socket clientSocket; + private long timeout = 2500; + private ZMQ.Poller poller; + + public MajordomoTestClientAsync(final String broker) { + this.broker = broker.replace("mdp://", "tcp://"); + ctx = new ZContext(); + reconnectToBroker(); + } + + public void destroy() { + ctx.destroy(); + } + + public long getTimeout() { + return timeout; + } + + /** + * Returns the reply message or NULL if there was no reply. Does not attempt + * to recover from a broker failure, this is not possible without storing + * all unanswered requests and resending them all… + * @return the MdpMessage + */ + public OpenCmwProtocol.MdpMessage recv() { + // Poll socket for a reply, with timeout + if (poller.poll(timeout * 1000) == -1) { + return null; // Interrupted + } + + if (poller.pollin(0)) { + return OpenCmwProtocol.MdpMessage.receive(clientSocket, false); + } + return null; + } + + /** + * Send request to broker and get reply by hook or crook Takes ownership of request message and destroys it when sent. + * + * @param service UTF-8 encoded service name byte array + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + */ + public boolean send(final byte[] service, final byte[]... msgs) { + final String topic = new String(service, StandardCharsets.UTF_8); + final byte[] rbacToken = msgs.length > 1 ? msgs[1] : null; + return new OpenCmwProtocol.MdpMessage(null, PROT_CLIENT, GET_REQUEST, service, "requestID".getBytes(StandardCharsets.UTF_8), URI.create(topic), msgs[0], "", rbacToken).send(clientSocket); + } + + /** + * Send subscription request to broker + * + * @param service UTF-8 encoded service name byte array + * @param rbacToken optional RBAC-token + */ + public boolean subscribe(final byte[] service, final byte[]... rbacToken) { + final String topic = new String(service, StandardCharsets.UTF_8); + final byte[] rbacTokenByte = rbacToken.length > 0 ? rbacToken[0] : null; + return new OpenCmwProtocol.MdpMessage(null, PROT_CLIENT, SUBSCRIBE, service, "requestID".getBytes(StandardCharsets.UTF_8), URI.create(topic), EMPTY_FRAME, "", rbacTokenByte).send(clientSocket); + } + + /** + * Send request to broker and get reply by hook or crook Takes ownership of request message and destroys it when sent. + * + * @param service UTF-8 encoded service name byte array + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + */ + public boolean send(final String service, final byte[]... msgs) { + return send(service.getBytes(StandardCharsets.UTF_8), msgs); + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + /** + * Connect or reconnect to broker + */ + void reconnectToBroker() { + if (clientSocket != null) { + clientSocket.close(); + } + clientSocket = ctx.createSocket(SocketType.DEALER); + clientSocket.setHWM(0); + clientSocket.setIdentity("clientV2".getBytes(StandardCharsets.UTF_8)); + clientSocket.connect(broker); + if (poller != null) { + poller.unregister(clientSocket); + poller.close(); + } + poller = ctx.createPoller(1); + poller.register(clientSocket, ZMQ.Poller.POLLIN); + LOGGER.atDebug().addArgument(broker).log("connecting to broker at: '{}'"); + } +} diff --git a/server/src/test/java/io/opencmw/server/MajordomoTestClientSync.java b/server/src/test/java/io/opencmw/server/MajordomoTestClientSync.java new file mode 100644 index 00000000..3b1791fb --- /dev/null +++ b/server/src/test/java/io/opencmw/server/MajordomoTestClientSync.java @@ -0,0 +1,134 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static io.opencmw.OpenCmwProtocol.Command.GET_REQUEST; +import static io.opencmw.OpenCmwProtocol.MdpSubProtocol.PROT_CLIENT; + +import java.lang.management.ManagementFactory; +import java.net.URI; +import java.util.Arrays; +import java.util.Formatter; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.ZMsg; + +import io.opencmw.OpenCmwProtocol; + +/** +* Majordomo Protocol Client API, implements the OpenCMW MDP variant +* +*/ +public class MajordomoTestClientSync { + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoTestClientSync.class); + private static final AtomicInteger CLIENT_V1_INSTANCE = new AtomicInteger(); + private final String uniqueID; + private final byte[] uniqueIdBytes; + private final String broker; + private final ZContext ctx; + private ZMQ.Socket clientSocket; + private long timeout = 2500; + private int retries = 3; + private final Formatter log = new Formatter(System.out); + private ZMQ.Poller poller; + + public MajordomoTestClientSync(String broker, String clientName) { + this.broker = broker.replace("mdp://", "tcp://"); + ctx = new ZContext(); + + uniqueID = clientName + "PID=" + ManagementFactory.getRuntimeMXBean().getName() + "-InstanceID=" + CLIENT_V1_INSTANCE.getAndIncrement(); + uniqueIdBytes = uniqueID.getBytes(ZMQ.CHARSET); + + reconnectToBroker(); + } + + /** + * Connect or reconnect to broker + */ + void reconnectToBroker() { + if (clientSocket != null) { + clientSocket.close(); + } + clientSocket = ctx.createSocket(SocketType.DEALER); + clientSocket.setHWM(0); + clientSocket.setIdentity(uniqueIdBytes); + clientSocket.connect(broker); + + if (poller != null) { + poller.unregister(clientSocket); + poller.close(); + } + poller = ctx.createPoller(1); + poller.register(clientSocket, ZMQ.Poller.POLLIN); + LOGGER.atDebug().addArgument(broker).log("connecting to broker at: '{}'"); + } + + public void destroy() { + ctx.destroy(); + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public String getUniqueID() { + return uniqueID; + } + + /** + * Send request to broker and get reply by hook or crook takes ownership of + * request message and destroys it when sent. Returns the reply message or + * NULL if there was no reply. + * + * @param service UTF-8 encoded service name + * @param msgs message(s) to be sent to OpenCmwProtocol broker (if more than one, than the last is assumed to be a RBAC-token + * @return reply message or NULL if there was no reply + */ + public OpenCmwProtocol.MdpMessage send(final String service, final byte[]... msgs) { + ZMsg reply = null; + + int retriesLeft = retries; + while (retriesLeft > 0 && !Thread.currentThread().isInterrupted()) { + final URI topic = URI.create(service); + final byte[] serviceBytes = StringUtils.stripStart(topic.getPath(), "/").getBytes(UTF_8); + final byte[] rbacToken = msgs.length > 1 ? msgs[1] : null; + if (!new OpenCmwProtocol.MdpMessage(null, PROT_CLIENT, GET_REQUEST, serviceBytes, "requestID".getBytes(UTF_8), topic, msgs[0], "", rbacToken).send(clientSocket)) { + throw new IllegalStateException("could not send request " + Arrays.toString(msgs)); + } + + // Poll socket for a reply, with timeout + if (poller.poll(timeout) == -1) + break; // Interrupted + + if (poller.pollin(0)) { + return OpenCmwProtocol.MdpMessage.receive(clientSocket, false); + } else { + if (--retriesLeft == 0) { + log.format("W: permanent error, abandoning\n"); + break; + } + log.format("W: no reply, reconnecting\n"); + reconnectToBroker(); + } + } + return null; + } +} diff --git a/server/src/test/java/io/opencmw/server/MajordomoWorkerTests.java b/server/src/test/java/io/opencmw/server/MajordomoWorkerTests.java new file mode 100644 index 00000000..b2eeaeb3 --- /dev/null +++ b/server/src/test/java/io/opencmw/server/MajordomoWorkerTests.java @@ -0,0 +1,409 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.*; + +import static io.opencmw.OpenCmwProtocol.MdpMessage; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.zeromq.SocketType; +import org.zeromq.Utils; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; +import org.zeromq.util.ZData; + +import io.opencmw.MimeType; +import io.opencmw.OpenCmwProtocol; +import io.opencmw.QueryParameterParser; +import io.opencmw.domain.BinaryData; +import io.opencmw.domain.NoData; +import io.opencmw.filter.TimingCtx; +import io.opencmw.rbac.BasicRbacRole; +import io.opencmw.serialiser.IoBuffer; +import io.opencmw.serialiser.IoClassSerialiser; +import io.opencmw.serialiser.annotations.MetaInfo; +import io.opencmw.serialiser.spi.BinarySerialiser; +import io.opencmw.serialiser.spi.ClassFieldDescription; +import io.opencmw.serialiser.spi.CmwLightSerialiser; +import io.opencmw.serialiser.spi.FastByteBuffer; +import io.opencmw.serialiser.spi.JsonSerialiser; + +/** + * Basic test for MajordomoWorker abstract class + * @author rstein + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MajordomoWorkerTests { + public static final String TEST_SERVICE_NAME = "basicHtml"; + private static final Logger LOGGER = LoggerFactory.getLogger(MajordomoWorkerTests.class); + private MajordomoBroker broker; + private TestHtmlService basicHtmlService; + private MajordomoTestClientSync clientSession; + + @Test + void testGetterSetter() { + assertDoesNotThrow(() -> new MajordomoWorker<>(broker.getContext(), "testServiceName", TestContext.class, RequestDataType.class, ReplyDataType.class, BasicRbacRole.ADMIN)); + assertDoesNotThrow(() -> new MajordomoWorker<>("mdp://localhost", "testServiceName", TestContext.class, RequestDataType.class, ReplyDataType.class, BasicRbacRole.ADMIN)); + + MajordomoWorker internal = new MajordomoWorker<>(broker.getContext(), "testServiceName", + TestContext.class, RequestDataType.class, ReplyDataType.class, BasicRbacRole.ADMIN); + + assertThrows(UnsupportedOperationException.class, () -> internal.registerHandler((rawCtx) -> {})); + + final MajordomoWorker.Handler handler = (rawCtx, reqCtx, in, repCtx, out) -> {}; + assertThat(internal.getHandler(), is(not(equalTo(handler)))); + assertDoesNotThrow(() -> internal.setHandler(handler)); + assertEquals(handler, internal.getHandler(), "handler get/set identity"); + + final MajordomoWorker.Handler htmlHandler = (rawCtx, reqCtx, in, repCtx, out) -> {}; + assertThat(internal.getHtmlHandler(), is(not(equalTo(handler)))); + assertDoesNotThrow(() -> internal.setHtmlHandler(htmlHandler)); + assertEquals(htmlHandler, internal.getHtmlHandler(), "HTML-handler get/set identity"); + } + + @Test + void simpleTest() throws IOException { + final MajordomoBroker broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final String brokerAddress = broker.bind("mdp://*:" + Utils.findOpenPort()); + final String brokerPubAddress = broker.bind("mds://*:" + Utils.findOpenPort()); + broker.start(); + + RequestDataType inputData = new RequestDataType(); + inputData.name = ""; + + final String requestTopic = "mdp://myServer:5555/helloWorld?ctx=FAIR.SELECTOR.C=2&testProperty=TestValue"; + final String testServiceName = URI.create(requestTopic).getPath(); + + // add external (albeit inproc) Majordomo worker to the broker + MajordomoWorker internal = new MajordomoWorker<>(broker.getContext(), testServiceName, + TestContext.class, RequestDataType.class, ReplyDataType.class, BasicRbacRole.ADMIN); + internal.setHandler((rawCtx, reqCtx, in, repCtx, out) -> { + LOGGER.atInfo().addArgument(reqCtx).log("received reqCtx = {}"); + LOGGER.atInfo().addArgument(in.name).log("received in.name = {}"); + + // processing data + out.name = in.name + "-modified"; + repCtx.ctx = TimingCtx.get("FAIR.SELECTOR.C=3"); + repCtx.ctx.bpcts = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); + + LOGGER.atInfo().addArgument(repCtx).log("received reqCtx = {}"); + LOGGER.atInfo().addArgument(out.name).log("received out.name = {}"); + repCtx.contentType = MimeType.BINARY; // set default return type to OpenCMW's YaS + }); + internal.start(); + + // using simple synchronous client + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + IoBuffer ioBuffer = new FastByteBuffer(4000); + IoClassSerialiser serialiser = new IoClassSerialiser(ioBuffer); + serialiser.serialiseObject(inputData); + byte[] input = Arrays.copyOf(ioBuffer.elements(), ioBuffer.position()); + { + final MdpMessage reply = clientSession.send(requestTopic, input); // w/o RBAC + if (!reply.errors.isBlank()) { + LOGGER.atError().addArgument(reply).log("reply with exceptions:\n{}"); + } + assertEquals("", reply.errors, "worker threw exception in reply"); + IoClassSerialiser deserialiser = new IoClassSerialiser(FastByteBuffer.wrap(reply.data)); + final ReplyDataType result = deserialiser.deserialiseObject(ReplyDataType.class); + + assertTrue(result.name.startsWith(inputData.name), "serialise-deserialise identity"); + } + + broker.stopBroker(); + } + + @Test + void testSerialiserIdentity() { + RequestDataType inputData = new RequestDataType(); + inputData.name = ""; + + IoBuffer ioBuffer = new FastByteBuffer(4000); + IoClassSerialiser serialiser = new IoClassSerialiser(ioBuffer); + serialiser.serialiseObject(inputData); + ioBuffer.flip(); + byte[] reply = Arrays.copyOf(ioBuffer.elements(), ioBuffer.limit() + 4); + + IoClassSerialiser deserialiser = new IoClassSerialiser(FastByteBuffer.wrap(reply)); + final RequestDataType result = deserialiser.deserialiseObject(RequestDataType.class); + assertTrue(result.name.startsWith(inputData.name), "serialise-deserialise identity"); + } + + @BeforeAll + void startBroker() throws IOException { + broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + final String brokerAddress = broker.bind("mdp://*:" + Utils.findOpenPort()); + broker.start(); + + basicHtmlService = new TestHtmlService(broker.getContext()); + basicHtmlService.start(); + clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + } + + @AfterAll + void stopBroker() { + broker.stopBroker(); + } + + @ParameterizedTest + @ValueSource(strings = { "", "BINARY", "JSON", "CMWLIGHT", "HTML" }) + void basicEchoTest(final String contentType) { + final BinaryData inputData = new BinaryData(); + inputData.resourceName = "test"; + inputData.contentType = MimeType.TEXT; + inputData.data = "testBinaryData".getBytes(UTF_8); + + IoBuffer ioBuffer = new FastByteBuffer(4000, true, null); + IoClassSerialiser serialiser = new IoClassSerialiser(ioBuffer); + serialiser.setAutoMatchSerialiser(false); + switch (contentType) { + case "HTML": + case "JSON": + serialiser.setMatchedIoSerialiser(JsonSerialiser.class); + serialiser.serialiseObject(inputData); + break; + case "CMWLIGHT": + serialiser.setMatchedIoSerialiser(CmwLightSerialiser.class); + serialiser.serialiseObject(inputData); + break; + case "": + case "BINARY": + default: + serialiser.setMatchedIoSerialiser(BinarySerialiser.class); + serialiser.serialiseObject(inputData); + break; + } + ioBuffer.flip(); + byte[] requestData = Arrays.copyOf(ioBuffer.elements(), ioBuffer.limit()); + final String mimeType = contentType.isBlank() ? "" : ("?contentType=" + contentType) + ("HTML".equals(contentType) ? "&noMenu" : ""); + final OpenCmwProtocol.MdpMessage rawReply = clientSession.send(TEST_SERVICE_NAME + mimeType, requestData); + assertNotNull(rawReply, "rawReply not being null"); + assertEquals("", rawReply.errors, "no exception thrown"); + assertNotNull(rawReply.data, "user-data not being null"); + + // test input/output equality + switch (contentType) { + case "HTML": + // HTML return + final String replyHtml = new String(rawReply.data); + // very crude check whether required reply field ids are present - we skip detailed HTLM parsing -> more efficiently done by a human and browser + assertThat(replyHtml, containsString("id=\"resourceName\"")); + assertThat(replyHtml, containsString("id=\"contentType\"")); + assertThat(replyHtml, containsString("id=\"data\"")); + assertThat(replyHtml, containsString(inputData.resourceName)); + assertThat(replyHtml, containsString(inputData.contentType.name())); + return; + case "": + case "JSON": + case "CMWLIGHT": + case "BINARY": + IoClassSerialiser deserialiser = new IoClassSerialiser(FastByteBuffer.wrap(rawReply.data)); + final BinaryData reply = deserialiser.deserialiseObject(BinaryData.class); + ioBuffer.flip(); + + assertNotNull(reply); + System.err.println("reply " + reply); + assertEquals(inputData.resourceName, reply.resourceName, "identity resourceName field"); + assertEquals(inputData.contentType, reply.contentType, "identity contentType field"); + assertArrayEquals(inputData.data, reply.data, "identity data field"); + return; + default: + throw new IllegalStateException("unimplemented contentType test: contentType=" + contentType); + } + } + + @Test + void basicDefaultHtmlHandlerTest() { + assertDoesNotThrow(() -> new DefaultHtmlHandler<>(this.getClass(), null, map -> { + map.put("extraKey", "extraValue"); + map.put("extraUnkownObject", new NoData()); + })); + } + + @Test + void testGenerateQueryParameter() { + final TestParameterClass testParam = new TestParameterClass(); + final Map map = DefaultHtmlHandler.generateQueryParameter(testParam); + assertNotNull(map); + } + + @Test + void testNotifySubscription() { + // start low-level subscription + final AtomicInteger subCounter = new AtomicInteger(0); + final AtomicBoolean run = new AtomicBoolean(true); + final AtomicBoolean startedSubscriber = new AtomicBoolean(false); + final Thread subcriptionThread = new Thread(() -> { + try (ZMQ.Socket sub = broker.getContext().createSocket(SocketType.SUB)) { + sub.setHWM(0); + sub.connect(MajordomoBroker.INTERNAL_ADDRESS_PUBLISHER); + sub.subscribe(TEST_SERVICE_NAME); + sub.subscribe(""); + while (run.get() && !Thread.interrupted()) { + startedSubscriber.set(true); + final MdpMessage rawReply = MdpMessage.receive(sub, false); + if (rawReply == null) { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + continue; + } + IoClassSerialiser deserialiser = new IoClassSerialiser(FastByteBuffer.wrap(rawReply.data)); + final BinaryData reply = deserialiser.deserialiseObject(BinaryData.class); + final Map> queryMap = QueryParameterParser.getMap(rawReply.topic.getQuery()); + final List testValues = queryMap.get("testValue"); + final int iteration = subCounter.getAndIncrement(); + + // very basic check that the correct notifications have been received in order + assertEquals("resourceName" + iteration, reply.resourceName, "resourceName field"); + assertEquals(1, testValues.size(), "test query parameter number"); + assertEquals("notify" + iteration, testValues.get(0), "test query parameter name"); + } + sub.unsubscribe(TEST_SERVICE_NAME); + } + }); + subcriptionThread.start(); + + // wait until all services are initialised + await().alias("wait for thread2 to start").atMost(1, TimeUnit.SECONDS).until(startedSubscriber::get, equalTo(true)); + + // send bursts of 10 messages + for (int i = 0; i < 10; i++) { + TestContext notifyCtx = new TestContext(); + notifyCtx.testValue = "notify" + i; + BinaryData reply = new BinaryData(); + reply.resourceName = "resourceName" + i; + basicHtmlService.notify(notifyCtx, reply); + } + + await().alias("wait for reply messages").atMost(2, TimeUnit.SECONDS).until(subCounter::get, equalTo(10)); + run.set(false); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(100)); + assertFalse(subcriptionThread.isAlive(), "subscription thread shut-down"); + assertEquals(10, subCounter.get(), "received expected number of subscription replies"); + } + + public static class TestContext { + @MetaInfo(description = "FAIR timing context selector, e.g. FAIR.SELECTOR.C=0, ALL, ...") + public TimingCtx ctx = TimingCtx.get("FAIR.SELECTOR.ALL"); + @MetaInfo(unit = "a.u.", description = "random test parameter") + public String testValue = "default value"; + @MetaInfo(description = "requested MIME content type, eg. 'application/binary', 'text/html','text/json', ..") + public MimeType contentType = MimeType.UNKNOWN; + + public TestContext() { + // needs default constructor + } + + @Override + public String toString() { + return "TestContext{ctx=" + ctx + ", testValue='" + testValue + "', contentType=" + contentType.getMediaType() + '}'; + } + } + + @MetaInfo(description = "request type class description", direction = "IN") + public static class RequestDataType { + @MetaInfo(description = " RequestDataType name to show up in the OpenAPI docs") + public String name; + public int counter; + public byte[] payload; + + public RequestDataType() { + // needs default constructor + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + final RequestDataType requestDataType = (RequestDataType) o; + + if (!Objects.equals(name, requestDataType.name)) + return false; + return Arrays.equals(payload, requestDataType.payload); + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + Arrays.hashCode(payload); + return result; + } + + @Override + public String toString() { + return "RequestDataType{inputName='" + name + "', counter=" + counter + "', payload=" + ZData.toString(payload) + '}'; + } + } + + @MetaInfo(description = "reply type class description", direction = "OUT") + public static class ReplyDataType { + @MetaInfo(description = "ReplyDataType name to show up in the OpenAPI docs") + public String name; + @MetaInfo(description = "a return value", unit = "A", direction = "OUT", groups = { "A", "B" }) + public int returnValue; + public ReplyDataType() { + // needs default constructor + } + + @Override + public String toString() { + return "ReplyDataType{outputName='" + name + "', returnValue=" + returnValue + '}'; + } + } + + @MetaInfo(description = "test HTML enabled MajordomoWorker implementation") + private static class TestHtmlService extends MajordomoWorker { + private TestHtmlService(ZContext ctx) { + super(ctx, TEST_SERVICE_NAME, MajordomoWorkerTests.TestContext.class, BinaryData.class, BinaryData.class); + setHtmlHandler(new DefaultHtmlHandler<>(this.getClass(), null, map -> { + map.put("extraKey", "extraValue"); + map.put("extraUnkownObject", new NoData()); + })); + super.setHandler((rawCtx, reqCtx, request, repCtx, reply) -> { + reply.data = request.data; + reply.contentType = request.contentType; + reply.resourceName = request.resourceName; + }); + } + } + + static class TestParameterClass { + public String testString = "test1"; + public Object genericObject = new Object(); + public UnknownClass ctx = new UnknownClass(); + } + + static class UnknownClass { + public String name = "UnknownClass"; + } +} diff --git a/server/src/test/java/io/opencmw/server/MmiServiceHelperTests.java b/server/src/test/java/io/opencmw/server/MmiServiceHelperTests.java new file mode 100644 index 00000000..043c33b1 --- /dev/null +++ b/server/src/test/java/io/opencmw/server/MmiServiceHelperTests.java @@ -0,0 +1,151 @@ +package io.opencmw.server; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.zeromq.Utils; +import org.zeromq.util.ZData; + +import io.opencmw.OpenCmwProtocol; +import io.opencmw.rbac.BasicRbacRole; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class MmiServiceHelperTests { + private static final String DEFAULT_REQUEST_MESSAGE = "Hello World!"; + private static final byte[] DEFAULT_REQUEST_MESSAGE_BYTES = DEFAULT_REQUEST_MESSAGE.getBytes(UTF_8); + private MajordomoBroker broker; + private String brokerAddress; + + @BeforeAll + void startBroker() throws IOException { + broker = new MajordomoBroker("TestBroker", "", BasicRbacRole.values()); + // broker.setDaemon(true); // use this if running in another app that controls threads + brokerAddress = broker.bind("mdp://*:" + Utils.findOpenPort()); + broker.start(); + } + + @AfterAll + void stopBroker() { + broker.stopBroker(); + } + + @Test + void basicEchoTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.echo", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, reply.data, "equal data"); + } + + @Test + void basicEchoHtmlTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.echo?contentType=HTML", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertArrayEquals(DEFAULT_REQUEST_MESSAGE_BYTES, reply.data, "equal data"); + } + + @Test + void basicDnsTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.dns", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("[TestBroker: mdp://")); + } + + @Test + void basicDnsHtmlTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.dns?contentType=HTML", DEFAULT_REQUEST_MESSAGE_BYTES); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("[TestBroker: mdp://")); + } + + @Test + void basicSerivceTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + { + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.service", "".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("mmi.dns,mmi.echo,mmi.openapi,mmi.service")); + } + + { + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.service", "mmi.service".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("200")); + } + + { + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.service", "doesNotExist".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("400")); + } + } + + @Test + void basicServiceHtmlTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + { + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.service?contentType=HTML", "".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("mmi.dns,mmi.echo,mmi.openapi,mmi.service")); + } + + { + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.service?contentType=HTML", "mmi.service".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("200")); + } + + { + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.service?contentType=HTML", "doesNotExist".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("400")); + } + } + + @Test + void basicOpenAPITest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.openapi", "mmi.echo".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertTrue(ZData.toString(reply.data).startsWith("io.opencmw.server.MmiServiceHelper$MmiEcho")); + } + + @Test + void basicOpenAPIExceptionTest() { + MajordomoTestClientSync clientSession = new MajordomoTestClientSync(brokerAddress, "customClientName"); + + final OpenCmwProtocol.MdpMessage reply = clientSession.send("mmi.openapi?contentType=HTML", "".getBytes(UTF_8)); // w/o RBAC + assertNotNull(reply, "reply not being null"); + assertNotNull(reply.data, "user-data not being null"); + assertNotNull(reply.errors); + assertFalse(reply.errors.isBlank()); + } +} diff --git a/server/src/test/resources/simplelogger.properties b/server/src/test/resources/simplelogger.properties new file mode 100644 index 00000000..4c11c945 --- /dev/null +++ b/server/src/test/resources/simplelogger.properties @@ -0,0 +1,50 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. + +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=debug + +# The output target which can be the path to a file, or the special values "System.out" and "System.err". +# Default is "System.err". +org.slf4j.simpleLogger.logFile=System.out + +# If the output target is set to "System.out" or "System.err" (see preceding entry), by default, +# logs will be output to the latest value referenced by System.out/err variables. +# By setting this parameter to true, the output stream will be cached, i.e. assigned once at initialization +# time and re-used independently of the current value referenced by System.out/err. +org.slf4j.simpleLogger.cacheOutputStream=true + +# Logging detail level for a SimpleLogger instance named "a.b.c". Right-side value must be one of +# "trace", "debug", "info", "warn", "error" or "off". When a SimpleLogger named "a.b.c" is initialized, +# its level is assigned from this property. If unspecified, the level of nearest parent logger will be used, +# and if none is set, then the value specified by org.slf4j.simpleLogger.defaultLogLevel will be used. +org.slf4j.simpleLogger.log.io.opencmw.*=debug + +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx= + +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false + +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z + +# Set to true if you want to output the current thread name. +# Defaults to true. +# org.slf4j.simpleLogger.showThreadName=false + +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true + +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +# org.slf4j.simpleLogger.showShortLogName=true \ No newline at end of file diff --git a/setupGitHooks.sh b/setupGitHooks.sh new file mode 100755 index 00000000..afa619a7 --- /dev/null +++ b/setupGitHooks.sh @@ -0,0 +1,4 @@ +#/bin/bash +# git config core.hooksPath $PWD/config/hooks +# safe fall-back for older git versions +ln -s -r ./config/hooks/pre* -t ./.git/hooks/