From 1882d7fe095f660c8973c7e456eff6c3c9fa9eb8 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Fri, 30 Sep 2022 18:06:01 +0400 Subject: [PATCH 01/40] [TH2-3942] Migrate to the latest core --- README.md | 42 +++----- build.gradle | 13 +-- .../java/com/exactpro/th2/FixHandler.java | 94 ++++++++++++------ .../com/exactpro/th2/FixHandlerContext.java | 52 ---------- .../com/exactpro/th2/FixHandlerFactory.java | 20 ++-- .../com/exactpro/th2/FixHandlerSettings.java | 35 ++++++- .../th2/conn/dirty/fix/FixByteBufUtil.kt | 18 ++-- .../th2/conn/dirty/fix/FixProtocolMangler.kt | 21 ++-- .../java/com/exactpro/th2/FixHandlerTest.java | 97 +++++++++++-------- .../conn/dirty/fix/TestMessageTransformer.kt | 1 - 10 files changed, 200 insertions(+), 193 deletions(-) delete mode 100644 src/main/java/com/exactpro/th2/FixHandlerContext.java diff --git a/README.md b/README.md index 24299be..3603286 100644 --- a/README.md +++ b/README.md @@ -4,34 +4,24 @@ This microservice allows sending and receiving messages via FIX protocol ## Configuration -+ *autoStart* - enables/disable auto-starting of session on box start (`true` by default) -+ *autoStopAfter* - time in seconds after which session will be automatically stopped (`0` by default = disabled) ++ *sessions* - list of session settings + *maxBatchSize* - max size of outgoing message batch (`1000` by default) + *maxFlushTime* - max message batch flush time (`1000` by default) ++ *batchByGroup* - batch messages by group instead of session alias and direction (`true` by default) + *publishSentEvents* - enables/disables publish of "message sent" events (`true` by default) + *publishConnectEvents* - enables/disables publish of "connect/disconnect" events (`true` by default) -+ *sessions* - list of session settings ## Session settings + *sessionAlias* - session alias for incoming/outgoing th2 messages -+ *host* - service host -+ *port* - service port -+ *security* - connection security settings -+ *maxMessageRate* - max outgoing message rate for this session (unlimited by default) -+ *autoReconnect* - enables/disables auto-reconnect (`true` by default) -+ *reconnectDelay* - delay between reconnects (`5000` by default) + *handler* - handler settings + *mangler* - mangler settings -### Security settings - -+ *ssl* - enables SSL on connection (`false` by default) -+ *sni* - enables SNI support (`false` by default) -+ *certFile* - path to server certificate (`null` by default) -+ *acceptAllCerts* - accept all server certificates (`false` by default, takes precedence over `certFile`) ## Handler settings ++ *host* - service host ++ *port* - service port ++ *security* - connection security settings + *beginString* - defines the start of a new message and the protocol version + *heartBtInt* - message waiting interval + *senderCompID* - ID of the sender of the message @@ -47,6 +37,13 @@ This microservice allows sending and receiving messages via FIX protocol + *resetSeqNumFlag* - resetting sequence number in initial Logon message (when conn started) + *resetOnLogon* - resetting the sequence number in Logon in other cases (e.g. disconnect) +### Security settings + ++ *ssl* - enables SSL on connection (`false` by default) ++ *sni* - enables SNI support (`false` by default) ++ *certFile* - path to server certificate (`null` by default) ++ *acceptAllCerts* - accept all server certificates (`false` by default, takes precedence over `certFile`) + ## Mangler settings Mangler is configured by specifying a list of transformations which it will try to apply to outgoing messages. @@ -217,10 +214,9 @@ spec: image-version: 0.0.1 type: th2-conn custom-config: - autoStart: true - autoStopAfter: 0 - maxBatchSize: 100 + maxBatchSize: 1000 maxFlushTime: 1000 + batchByGroup: false publishSentEvents: true publishConnectEvents: true sessions: @@ -278,17 +274,11 @@ spec: settings: storageOnDemand: false queueLength: 1000 - - name: outgoing_messages - connection-type: mq - attributes: - - second - - publish - - raw - - name: incoming_messages + - name: outgoing connection-type: mq attributes: - - first - publish + - store - raw extended-settings: externalBox: diff --git a/build.gradle b/build.gradle index 6e0bc57..20d519c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { id 'java' - id 'org.jetbrains.kotlin.jvm' version '1.5.31' + id 'org.jetbrains.kotlin.jvm' version '1.6.21' id 'com.palantir.docker' version '0.25.0' } @@ -38,14 +38,15 @@ repositories { dependencies { api platform('com.exactpro.th2:bom:3.1.0') - implementation 'com.exactpro.th2:conn-dirty-tcp-core:1.0.0' - implementation 'com.exactpro.th2:common:3.32.0' + implementation 'com.exactpro.th2:common:3.40.0-TH2-3789-2817159492-SNAPSHOT' + implementation 'com.exactpro.th2:netty-bytebuf-utils:0.0.1' + implementation 'com.exactpro.th2:conn-dirty-tcp-core:2.0.0' implementation 'org.slf4j:slf4j-api:1.7.33' - implementation 'io.github.microutils:kotlin-logging:2.1.21' + implementation 'io.github.microutils:kotlin-logging:2.1.23' implementation 'io.netty:netty-all:4.1.72.Final' - implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.0' + implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.21' implementation 'com.google.auto.service:auto-service:1.0.1' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: jackson_version @@ -54,7 +55,7 @@ dependencies { implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jackson_version testImplementation 'org.mockito:mockito-all:1.10.19' - testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5:1.6.0' + testImplementation 'org.jetbrains.kotlin:kotlin-test-junit5:1.7.10' annotationProcessor 'com.google.auto.service:auto-service:1.0.1' kapt 'com.google.auto.service:auto-service:1.0.1' diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 1a117df..c1e245c 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -16,20 +16,23 @@ package com.exactpro.th2; +import com.exactpro.th2.common.grpc.MessageID; +import com.exactpro.th2.common.grpc.RawMessage; import com.exactpro.th2.conn.dirty.fix.FixField; import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel; -import com.exactpro.th2.conn.dirty.tcp.core.api.IContext; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandler; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandlerSettings; +import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel.SendMode; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandler; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerContext; import com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil; -import com.google.auto.service.AutoService; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -51,8 +54,8 @@ import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.lastField; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.updateChecksum; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.updateLength; -import static com.exactpro.th2.conn.dirty.tcp.core.util.ByteBufUtil.indexOf; -import static com.exactpro.th2.conn.dirty.tcp.core.util.ByteBufUtil.isEmpty; +import static com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil.getEventId; +import static com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil.toByteBuf; import static com.exactpro.th2.constants.Constants.BEGIN_SEQ_NO; import static com.exactpro.th2.constants.Constants.BEGIN_SEQ_NO_TAG; import static com.exactpro.th2.constants.Constants.BEGIN_STRING_TAG; @@ -93,6 +96,8 @@ import static com.exactpro.th2.constants.Constants.TEST_REQ_ID_TAG; import static com.exactpro.th2.constants.Constants.TEXT_TAG; import static com.exactpro.th2.constants.Constants.USERNAME; +import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.indexOf; +import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.isEmpty; import static com.exactpro.th2.util.MessageUtil.findByte; import static java.nio.charset.StandardCharsets.US_ASCII; @@ -101,45 +106,70 @@ //todo ring buffer as cache //todo add events -@AutoService(IProtocolHandler.class) -public class FixHandler implements AutoCloseable, IProtocolHandler { - +public class FixHandler implements AutoCloseable, IHandler { private static final Logger LOGGER = LoggerFactory.getLogger(FixHandler.class); private static final String SOH = "\001"; private static final byte BYTE_SOH = 1; private static final String STRING_MSG_TYPE = "MsgType"; private static final String REJECT_REASON = "Reject reason"; private static final String STUBBING_VALUE = "XXX"; + private final Log outgoingMessages = new Log(10000); private final AtomicInteger msgSeqNum = new AtomicInteger(0); private final AtomicInteger serverMsgSeqNum = new AtomicInteger(0); private final AtomicInteger testReqID = new AtomicInteger(0); private final AtomicBoolean enabled = new AtomicBoolean(false); private final AtomicBoolean connStarted = new AtomicBoolean(false); - private final ScheduledExecutorService executorService; - private final IContext context; + private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); + private final IHandlerContext context; + private final InetSocketAddress address; + private Future heartbeatTimer = CompletableFuture.completedFuture(null); private Future testRequestTimer = CompletableFuture.completedFuture(null); private Future reconnectRequestTimer = CompletableFuture.completedFuture(null); private Future disconnectRequest = CompletableFuture.completedFuture(null); - private IChannel client; + private volatile IChannel channel; protected FixHandlerSettings settings; private long lastSendTime = System.currentTimeMillis(); - public FixHandler(IContext context) { + public FixHandler(IHandlerContext context) { this.context = context; - executorService = Executors.newScheduledThreadPool(1); this.settings = (FixHandlerSettings) context.getSettings(); + String host = settings.getHost(); + if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); + int port = settings.getPort(); + if (port < 1 || port > 65535) throw new IllegalArgumentException("port must be in 1..65535 range"); + address = new InetSocketAddress(host, port); + Objects.requireNonNull(settings.getSecurity(), "security cannot be null"); Objects.requireNonNull(settings.getBeginString(), "BeginString can not be null"); Objects.requireNonNull(settings.getResetSeqNumFlag(), "ResetSeqNumFlag can not be null"); Objects.requireNonNull(settings.getResetOnLogon(), "ResetOnLogon can not be null"); - if(settings.getHeartBtInt() <= 0) throw new IllegalArgumentException("HeartBtInt cannot be negative or zero"); - if(settings.getTestRequestDelay() <= 0) throw new IllegalArgumentException("TestRequestDelay cannot be negative or zero"); - if(settings.getDisconnectRequestDelay() <= 0) throw new IllegalArgumentException("DisconnectRequestDelay cannot be negative or zero"); + if (settings.getHeartBtInt() <= 0) throw new IllegalArgumentException("HeartBtInt cannot be negative or zero"); + if (settings.getTestRequestDelay() <= 0) throw new IllegalArgumentException("TestRequestDelay cannot be negative or zero"); + if (settings.getDisconnectRequestDelay() <= 0) throw new IllegalArgumentException("DisconnectRequestDelay cannot be negative or zero"); + } + + @Override + public void onStart() { + channel = context.createChannel(address, settings.getSecurity(), Map.of(), true, settings.getReconnectDelay() * 1000L, Integer.MAX_VALUE); + } + + @NotNull + @Override + public CompletableFuture send(@NotNull RawMessage rawMessage) { + if (!channel.isOpen()) { + try { + channel.open().get(); + } catch (Exception e) { + ExceptionUtils.rethrow(e); + } + } + + return channel.send(toByteBuf(rawMessage.getBody()), rawMessage.getMetadata().getPropertiesMap(), getEventId(rawMessage), SendMode.HANDLE_AND_MANGLE); } @Override - public ByteBuf onReceive(ByteBuf buffer) { + public ByteBuf onReceive(IChannel channel, ByteBuf buffer) { int offset = buffer.readerIndex(); if (offset == buffer.writerIndex()) return null; @@ -176,7 +206,7 @@ public ByteBuf onReceive(ByteBuf buffer) { @NotNull @Override - public Map onIncoming(@NotNull ByteBuf message) { + public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBuf message) { Map metadata = new HashMap<>(); int beginString = indexOf(message, "8=FIX"); @@ -289,7 +319,7 @@ public void sendResendRequest(int beginSeqNo, int endSeqNo) { //do private resendRequest.append(BEGIN_SEQ_NO).append(beginSeqNo).append(SOH); resendRequest.append(END_SEQ_NO).append(endSeqNo).append(SOH); setChecksumAndBodyLength(resendRequest); - client.send(Unpooled.wrappedBuffer(resendRequest.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), IChannel.SendMode.MANGLE); + channel.send(Unpooled.wrappedBuffer(resendRequest.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } void sendResendRequest(int beginSeqNo) { //do private @@ -301,7 +331,7 @@ void sendResendRequest(int beginSeqNo) { //do private setChecksumAndBodyLength(resendRequest); if (enabled.get()) { - client.send(Unpooled.wrappedBuffer(resendRequest.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), IChannel.SendMode.MANGLE); + channel.send(Unpooled.wrappedBuffer(resendRequest.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } else { sendLogon(); } @@ -331,10 +361,10 @@ private void handleResendRequest(ByteBuf message) { StringBuilder heartbeat = new StringBuilder(); setHeader(heartbeat, MSG_TYPE_HEARTBEAT, i); setChecksumAndBodyLength(heartbeat); - client.send(Unpooled.wrappedBuffer(heartbeat.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), IChannel.SendMode.MANGLE); + channel.send(Unpooled.wrappedBuffer(heartbeat.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } else { if (LOGGER.isInfoEnabled()) LOGGER.info("Resending message: {}", storedMsg.toString(US_ASCII)); - client.send(storedMsg, Collections.emptyMap(), IChannel.SendMode.MANGLE); + channel.send(storedMsg, Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } } } catch (Exception e) { @@ -351,7 +381,7 @@ private void sendSequenceReset() { setChecksumAndBodyLength(sequenceReset); if (enabled.get()) { - client.send(Unpooled.wrappedBuffer(sequenceReset.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), IChannel.SendMode.MANGLE); + channel.send(Unpooled.wrappedBuffer(sequenceReset.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } else { sendLogon(); } @@ -384,7 +414,7 @@ private boolean checkLogon(ByteBuf message) { @NotNull @Override - public void onOutgoing(@NotNull ByteBuf message, @NotNull Map metadata) { + public void onOutgoing(@NotNull IChannel channel, @NotNull ByteBuf message, @NotNull Map metadata) { lastSendTime = System.currentTimeMillis(); onOutgoingUpdateTag(message, metadata); @@ -495,8 +525,8 @@ public void onOutgoingUpdateTag(@NotNull ByteBuf message, @NotNull Map enabled.set(false), settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); } diff --git a/src/main/java/com/exactpro/th2/FixHandlerContext.java b/src/main/java/com/exactpro/th2/FixHandlerContext.java deleted file mode 100644 index 639cbd8..0000000 --- a/src/main/java/com/exactpro/th2/FixHandlerContext.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2022-2022 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.exactpro.th2; - -import com.exactpro.th2.common.event.Event; -import com.exactpro.th2.common.schema.dictionary.DictionaryType; -import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel; -import com.exactpro.th2.conn.dirty.tcp.core.api.IContext; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandlerSettings; -import com.google.auto.service.AutoService; -import org.jetbrains.annotations.NotNull; - -import java.io.InputStream; - -@AutoService(IContext.class) -public class FixHandlerContext implements IContext { - - @NotNull - @Override - public IChannel getChannel() { - return null; - } - - @Override - public IProtocolHandlerSettings getSettings() { - return new FixHandlerSettings(); - } - - @NotNull - @Override - public InputStream get(@NotNull DictionaryType dictionaryType) { - return null; - } - - @Override - public void send(@NotNull Event event) { - } -} diff --git a/src/main/java/com/exactpro/th2/FixHandlerFactory.java b/src/main/java/com/exactpro/th2/FixHandlerFactory.java index c870314..a11165f 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerFactory.java +++ b/src/main/java/com/exactpro/th2/FixHandlerFactory.java @@ -16,19 +16,18 @@ package com.exactpro.th2; -import com.exactpro.th2.conn.dirty.tcp.core.api.IContext; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandler; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandlerFactory; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandlerSettings; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandler; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerContext; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerFactory; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerSettings; import com.google.auto.service.AutoService; import org.jetbrains.annotations.NotNull; -@AutoService(IProtocolHandlerFactory.class) -public class FixHandlerFactory implements IProtocolHandlerFactory { - +@AutoService(IHandlerFactory.class) +public class FixHandlerFactory implements IHandlerFactory { @NotNull @Override - public Class getSettings() { + public Class getSettings() { return FixHandlerSettings.class; } @@ -38,9 +37,10 @@ public String getName() { return FixHandlerFactory.class.getSimpleName(); } + @NotNull @Override - public IProtocolHandler create(@NotNull IContext iContext) { - return new FixHandler(iContext); + public IHandler create(@NotNull IHandlerContext context) { + return new FixHandler(context); } } diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index fccc9f3..70fdee9 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -16,12 +16,13 @@ package com.exactpro.th2; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandlerSettings; -import com.google.auto.service.AutoService; - -@AutoService(IProtocolHandlerSettings.class) -public class FixHandlerSettings implements IProtocolHandlerSettings { +import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel.Security; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerSettings; +public class FixHandlerSettings implements IHandlerSettings { + private String host = null; + private int port = 0; + private Security security = new Security(); private String beginString = "FIXT.1.1"; private long heartBtInt = 30; private String senderCompID; @@ -37,6 +38,30 @@ public class FixHandlerSettings implements IProtocolHandlerSettings { private int reconnectDelay = 5; private int disconnectRequestDelay = 5; + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public Security getSecurity() { + return security; + } + + public void setSecurity(Security security) { + this.security = security; + } + public String getBeginString() { return beginString; } diff --git a/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixByteBufUtil.kt b/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixByteBufUtil.kt index 291939f..7b6548c 100644 --- a/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixByteBufUtil.kt +++ b/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixByteBufUtil.kt @@ -16,15 +16,15 @@ package com.exactpro.th2.conn.dirty.fix -import com.exactpro.th2.conn.dirty.tcp.core.util.EMPTY_STRING -import com.exactpro.th2.conn.dirty.tcp.core.util.endsWith -import com.exactpro.th2.conn.dirty.tcp.core.util.get -import com.exactpro.th2.conn.dirty.tcp.core.util.indexOf -import com.exactpro.th2.conn.dirty.tcp.core.util.insert -import com.exactpro.th2.conn.dirty.tcp.core.util.lastIndexOf -import com.exactpro.th2.conn.dirty.tcp.core.util.replace -import com.exactpro.th2.conn.dirty.tcp.core.util.requireReadable -import com.exactpro.th2.conn.dirty.tcp.core.util.subsequence +import com.exactpro.th2.netty.bytebuf.util.EMPTY_STRING +import com.exactpro.th2.netty.bytebuf.util.endsWith +import com.exactpro.th2.netty.bytebuf.util.get +import com.exactpro.th2.netty.bytebuf.util.indexOf +import com.exactpro.th2.netty.bytebuf.util.insert +import com.exactpro.th2.netty.bytebuf.util.lastIndexOf +import com.exactpro.th2.netty.bytebuf.util.replace +import com.exactpro.th2.netty.bytebuf.util.requireReadable +import com.exactpro.th2.netty.bytebuf.util.subsequence import io.netty.buffer.ByteBuf import java.nio.charset.Charset import kotlin.text.Charsets.UTF_8 diff --git a/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixProtocolMangler.kt b/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixProtocolMangler.kt index e734fd9..e491866 100644 --- a/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixProtocolMangler.kt +++ b/src/main/kotlin/com/exactpro/th2/conn/dirty/fix/FixProtocolMangler.kt @@ -21,10 +21,11 @@ import com.exactpro.th2.common.event.Event.Status.PASSED import com.exactpro.th2.common.event.EventUtils.createMessageBean import com.exactpro.th2.common.event.bean.IRow import com.exactpro.th2.common.event.bean.builder.TableBuilder -import com.exactpro.th2.conn.dirty.tcp.core.api.IContext -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolMangler -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolManglerFactory -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolManglerSettings +import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel +import com.exactpro.th2.conn.dirty.tcp.core.api.IMangler +import com.exactpro.th2.conn.dirty.tcp.core.api.IManglerContext +import com.exactpro.th2.conn.dirty.tcp.core.api.IManglerFactory +import com.exactpro.th2.conn.dirty.tcp.core.api.IManglerSettings import com.fasterxml.jackson.dataformat.yaml.YAMLMapper import com.fasterxml.jackson.module.kotlin.KotlinModule import com.fasterxml.jackson.module.kotlin.readValue @@ -42,10 +43,10 @@ private val MAPPER = YAMLMapper.builder() private const val RULE_NAME_PROPERTY = "rule-name" private const val RULE_ACTIONS_PROPERTY = "rule-actions" -class FixProtocolMangler(context: IContext) : IProtocolMangler { +class FixProtocolMangler(context: IManglerContext) : IMangler { private val rules = (context.settings as FixProtocolManglerSettings).rules - override fun onOutgoing(message: ByteBuf, metadata: MutableMap): Event? { + override fun onOutgoing(channel: IChannel, message: ByteBuf, metadata: MutableMap): Event? { LOGGER.trace { "Processing message: ${message.toString(Charsets.UTF_8)}" } val (rule, unconditionally) = getRule(message, metadata) ?: return null @@ -102,14 +103,14 @@ class FixProtocolMangler(context: IContext) : IProtoco } } -@AutoService(IProtocolManglerFactory::class) -class FixProtocolManglerFactory : IProtocolManglerFactory { +@AutoService(IManglerFactory::class) +class FixProtocolManglerFactory : IManglerFactory { override val name = "demo-fix-mangler" override val settings = FixProtocolManglerSettings::class.java - override fun create(context: IContext) = FixProtocolMangler(context) + override fun create(context: IManglerContext) = FixProtocolMangler(context) } -class FixProtocolManglerSettings(val rules: List = emptyList()) : IProtocolManglerSettings +class FixProtocolManglerSettings(val rules: List = emptyList()) : IManglerSettings private data class ActionRow( val corruptionType: String, diff --git a/src/test/java/com/exactpro/th2/FixHandlerTest.java b/src/test/java/com/exactpro/th2/FixHandlerTest.java index 0933d07..3d02568 100644 --- a/src/test/java/com/exactpro/th2/FixHandlerTest.java +++ b/src/test/java/com/exactpro/th2/FixHandlerTest.java @@ -16,14 +16,14 @@ package com.exactpro.th2; +import com.exactpro.th2.common.grpc.EventID; import com.exactpro.th2.common.grpc.MessageID; import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel; -import com.exactpro.th2.conn.dirty.tcp.core.api.IContext; -import com.exactpro.th2.conn.dirty.tcp.core.api.IProtocolHandlerSettings; -import com.exactpro.th2.conn.dirty.tcp.core.api.impl.Channel.Security; +import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerContext; import com.exactpro.th2.util.MessageUtil; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import kotlin.Unit; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -41,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import static com.exactpro.th2.constants.Constants.BEGIN_STRING_TAG; import static com.exactpro.th2.constants.Constants.BODY_LENGTH_TAG; @@ -55,15 +54,15 @@ class FixHandlerTest { - private static final Client client = new Client(); + private static final Channel channel = new Channel(); private static ByteBuf buffer; private static ByteBuf oneMessageBuffer; private static ByteBuf brokenBuffer; - private static FixHandler fixHandler = client.getFixHandler(); + private static FixHandler fixHandler = channel.getFixHandler(); @BeforeAll static void init() { - fixHandler.onOpen(); + fixHandler.onOpen(channel); oneMessageBuffer = Unpooled.wrappedBuffer("8=FIXT.1.1\0019=13\00135=AE\001552=1\00110=169\001".getBytes(StandardCharsets.US_ASCII)); buffer = Unpooled.wrappedBuffer(("8=FIXT.1.1\0019=13\00135=AE\001552=1\00110=169\0018=FIXT.1.1\0019=13\00135=NN" + "\001552=2\00110=100\0018=FIXT.1.1\0019=13\00135=NN\001552=2\00110=100\001").getBytes(StandardCharsets.US_ASCII)); @@ -85,18 +84,18 @@ void test3188(){ String body1 = "8=F"; String body2 = "IXT.1.1\0019=13\00135=AE\001552=1\00158=11111\00110=169\001"; ByteBuf byteBuf1 = Unpooled.buffer().writeBytes(body1.getBytes(StandardCharsets.UTF_8)); - fixHandler.onReceive(byteBuf1); + fixHandler.onReceive(channel, byteBuf1); assertEquals("8=F", byteBuf1.toString(StandardCharsets.US_ASCII)); byteBuf1.writeBytes(body2.getBytes(StandardCharsets.UTF_8)); - fixHandler.onReceive(byteBuf1); + fixHandler.onReceive(channel, byteBuf1); assertEquals("", byteBuf1.toString(StandardCharsets.US_ASCII)); } @Test void onDataBrokenMessageTest() { - ByteBuf result0 = fixHandler.onReceive(brokenBuffer); - ByteBuf result1 = fixHandler.onReceive(brokenBuffer); - ByteBuf result2 = fixHandler.onReceive(brokenBuffer); + ByteBuf result0 = fixHandler.onReceive(channel, brokenBuffer); + ByteBuf result1 = fixHandler.onReceive(channel, brokenBuffer); + ByteBuf result2 = fixHandler.onReceive(channel, brokenBuffer); String expected0 = "A"; assertNotNull(result0); @@ -111,9 +110,9 @@ void onReceiveCorrectMessagesTest() { buffer = Unpooled.wrappedBuffer(("8=FIXT.1.1\0019=13\00135=AE\001552=1\00158=11111\00110=169\0018=FIXT.1.1\0019=13\00135=NN" + "\001552=2\00110=100\0018=FIXT.1.1\0019=13\00135=NN\001552=2\00110=100\001").getBytes(StandardCharsets.US_ASCII)); - ByteBuf result0 = fixHandler.onReceive(buffer); - ByteBuf result1 = fixHandler.onReceive(buffer); - ByteBuf result2 = fixHandler.onReceive(buffer); + ByteBuf result0 = fixHandler.onReceive(channel, buffer); + ByteBuf result1 = fixHandler.onReceive(channel, buffer); + ByteBuf result2 = fixHandler.onReceive(channel, buffer); String expected1 = "8=FIXT.1.1\0019=13\00135=AE\001552=1\00158=11111\00110=169\001"; String expected2 = "8=FIXT.1.1\0019=13\00135=NN\001552=2\00110=100\001"; @@ -137,25 +136,25 @@ void sendResendRequestTest() { String expectedResendRequest = "8=FIXT.1.1\u00019=73\u000135=2\u000134=3\u000149=client\u000156=server" + // #2 sent resendRequest "\u000150=trader\u000152=2014-12-22T10:15:30Z\u00017=1\u000116=0\u000110=227\u0001"; - client.clearQueue(); + channel.clearQueue(); fixHandler.sendLogon(); fixHandler.sendResendRequest(1); - assertEquals(expectedLogon, new String(client.getQueue().get(0).array())); + assertEquals(expectedLogon, new String(channel.getQueue().get(0).array())); //assertEquals(expectedHeartbeat, new String(client.getQueue().get(1).array())); - assertEquals(expectedResendRequest, new String(client.getQueue().get(1).array())); + assertEquals(expectedResendRequest, new String(channel.getQueue().get(1).array())); } @Test void onConnectionTest() { - client.clearQueue(); - fixHandler.onOpen(); + channel.clearQueue(); + fixHandler.onOpen(channel); try { Thread.sleep(10000); } catch (InterruptedException e) { e.printStackTrace(); } assertEquals("8=FIXT.1.1\u00019=105\u000135=A\u000134=7\u000149=client\u000156=server\u000150=trader\u000152=2014-12-22T10:15:30Z\u000198=0\u0001108=30\u00011137=9\u0001553=username\u0001554=pass\u000110=209\u0001", - new String(client.getQueue().get(0).array())); + new String(channel.getQueue().get(0).array())); } @Test @@ -178,15 +177,15 @@ void onOutgoingMessageTest() { expected4.put("MsgType", "A"); Map actual = new HashMap<>(expected); - fixHandler.onOutgoing(bufferForPrepareMessage1, actual); + fixHandler.onOutgoing(channel, bufferForPrepareMessage1, actual); assertEquals(expected, actual); Map actual2 = new HashMap<>(); - fixHandler.onOutgoing(bufferForPrepareMessage2, actual2); + fixHandler.onOutgoing(channel, bufferForPrepareMessage2, actual2); assertEquals(expected2, actual2); - fixHandler.onOutgoing(bufferForPrepareMessage3, expected3); - fixHandler.onOutgoing(bufferForPrepareMessage4, expected4); + fixHandler.onOutgoing(channel, bufferForPrepareMessage3, expected3); + fixHandler.onOutgoing(channel, bufferForPrepareMessage4, expected4); bufferForPrepareMessage1.readerIndex(0); bufferForPrepareMessage2.readerIndex(0); @@ -232,9 +231,9 @@ void getByteByfBodyLengthTest() { @Test void sendTestRequestTest() { String expected = "8=FIXT.1.1\u00019=70\u000135=1\u000134=1\u000149=client\u000156=server\u000150=trader\u000152=2014-12-22T10:15:30Z\u0001112=1\u000110=101\u0001"; - client.clearQueue(); + channel.clearQueue(); fixHandler.sendTestRequest(); - assertEquals(expected, new String(client.getQueue().get(0).array())); + assertEquals(expected, new String(channel.getQueue().get(0).array())); } @Test @@ -318,13 +317,15 @@ void updateTagTest() { } -class Client implements IChannel { +class Channel implements IChannel { private final FixHandlerSettings fixHandlerSettings; private final MyFixHandler fixHandler; private final List queue = new ArrayList<>(); - Client() { + Channel() { this.fixHandlerSettings = new FixHandlerSettings(); + fixHandlerSettings.setHost("127.0.0.1"); + fixHandlerSettings.setPort(8080); fixHandlerSettings.setBeginString("FIXT.1.1"); fixHandlerSettings.setHeartBtInt(30); fixHandlerSettings.setSenderCompID("client"); @@ -339,26 +340,20 @@ class Client implements IChannel { fixHandlerSettings.setResetOnLogon(false); fixHandlerSettings.setDefaultApplVerID("9"); fixHandlerSettings.setSenderSubID("trader"); - IContext context = Mockito.mock(IContext.class); + IHandlerContext context = Mockito.mock(IHandlerContext.class); Mockito.when(context.getSettings()).thenReturn(fixHandlerSettings); - Mockito.when(context.getChannel()).thenReturn(this); this.fixHandler = new MyFixHandler(context); } @Override - public void open() { - - } - - @Override - public void open(InetSocketAddress address, Security security) { - + public CompletableFuture open() { + return CompletableFuture.completedFuture(Unit.INSTANCE); } @NotNull @Override - public Future send(@NotNull ByteBuf byteBuf, @NotNull Map map, @NotNull IChannel.SendMode sendMode) { + public CompletableFuture send(@NotNull ByteBuf byteBuf, @NotNull Map map, EventID eventId, @NotNull IChannel.SendMode sendMode) { queue.add(byteBuf); return CompletableFuture.completedFuture(MessageID.getDefaultInstance()); } @@ -369,8 +364,8 @@ public boolean isOpen() { } @Override - public void close() { - + public CompletableFuture close() { + return CompletableFuture.completedFuture(Unit.INSTANCE); } public FixHandlerSettings getFixHandlerSettings() { @@ -399,11 +394,29 @@ public InetSocketAddress getAddress() { public Security getSecurity() { return new Security(); } + + @NotNull + @Override + public Map getAttributes() { + return Map.of(); + } + + @NotNull + @Override + public String getSessionAlias() { + return "alias"; + } + + @NotNull + @Override + public String getSessionGroup() { + return "group"; + } } class MyFixHandler extends FixHandler { - public MyFixHandler(IContext context) { + public MyFixHandler(IHandlerContext context) { super(context); } diff --git a/src/test/kotlin/com/exactpro/th2/conn/dirty/fix/TestMessageTransformer.kt b/src/test/kotlin/com/exactpro/th2/conn/dirty/fix/TestMessageTransformer.kt index f761318..8732e90 100644 --- a/src/test/kotlin/com/exactpro/th2/conn/dirty/fix/TestMessageTransformer.kt +++ b/src/test/kotlin/com/exactpro/th2/conn/dirty/fix/TestMessageTransformer.kt @@ -101,7 +101,6 @@ class TestMessageTransformer { private infix fun Int.eq(value: String) = field(this, value) private infix fun Int.to(value: String) = field(this, value) private infix fun Int.oneOf(value: List) = FieldDefinition(this, null, null, value) - private infix fun List.oneOf(value: List) = FieldDefinition(null, null, this, value) private infix fun Int.matches(pattern: String) = select(this, pattern) private infix fun Int.matching(pattern: String) = select(this, pattern) private fun set(field: FieldDefinition) = Action(set = field) From 586aeb59e371da37079dbbd032e6a620a730826f Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Tue, 17 Jan 2023 19:31:56 +0400 Subject: [PATCH 02/40] Session managament based on NextExpSeq --- .../java/com/exactpro/th2/FixHandler.java | 149 +++++++++++++++--- .../com/exactpro/th2/FixHandlerSettings.java | 40 ++++- .../java/com/exactpro/th2/SequenceHolder.java | 34 ++++ .../com/exactpro/th2/constants/Constants.java | 18 +++ src/main/java/com/exactpro/th2/util/Util.java | 43 +++++ .../java/com/exactpro/th2/FixHandlerTest.java | 18 ++- 6 files changed, 275 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/exactpro/th2/SequenceHolder.java create mode 100644 src/main/java/com/exactpro/th2/util/Util.java diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 016d31d..120b86c 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -24,23 +24,33 @@ import com.exactpro.th2.conn.dirty.tcp.core.api.IHandler; import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerContext; import com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil; +import com.exactpro.th2.util.Util; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.checkerframework.checker.units.qual.A; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.TimeZone; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -48,6 +58,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.findField; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.findLastField; @@ -58,6 +69,7 @@ import static com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil.getEventId; import static com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil.toByteBuf; import static com.exactpro.th2.conn.dirty.fix.KeyFileType.Companion.OperationMode.ENCRYPT_MODE; +import static com.exactpro.th2.constants.Constants.ADMIN_MESSAGES; import static com.exactpro.th2.constants.Constants.BEGIN_SEQ_NO; import static com.exactpro.th2.constants.Constants.BEGIN_SEQ_NO_TAG; import static com.exactpro.th2.constants.Constants.BEGIN_STRING_TAG; @@ -72,6 +84,7 @@ import static com.exactpro.th2.constants.Constants.END_SEQ_NO_TAG; import static com.exactpro.th2.constants.Constants.GAP_FILL_FLAG_TAG; import static com.exactpro.th2.constants.Constants.HEART_BT_INT; +import static com.exactpro.th2.constants.Constants.IS_POSS_DUP; import static com.exactpro.th2.constants.Constants.MSG_SEQ_NUM; import static com.exactpro.th2.constants.Constants.MSG_SEQ_NUM_TAG; import static com.exactpro.th2.constants.Constants.MSG_TYPE; @@ -86,7 +99,10 @@ import static com.exactpro.th2.constants.Constants.NEW_PASSWORD; import static com.exactpro.th2.constants.Constants.NEW_SEQ_NO; import static com.exactpro.th2.constants.Constants.NEW_SEQ_NO_TAG; +import static com.exactpro.th2.constants.Constants.NEXT_EXPECTED_SEQ_NUM; +import static com.exactpro.th2.constants.Constants.NEXT_EXPECTED_SEQ_NUMBER_TAG; import static com.exactpro.th2.constants.Constants.PASSWORD; +import static com.exactpro.th2.constants.Constants.POSS_DUP_TAG; import static com.exactpro.th2.constants.Constants.RESET_SEQ_NUM; import static com.exactpro.th2.constants.Constants.SENDER_COMP_ID; import static com.exactpro.th2.constants.Constants.SENDER_COMP_ID_TAG; @@ -102,7 +118,9 @@ import static com.exactpro.th2.constants.Constants.TEXT_TAG; import static com.exactpro.th2.constants.Constants.USERNAME; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.indexOf; +import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.insert; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.isEmpty; +import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.set; import static com.exactpro.th2.util.MessageUtil.findByte; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.util.Objects.requireNonNull; @@ -121,14 +139,15 @@ public class FixHandler implements AutoCloseable, IHandler { private static final String STUBBING_VALUE = "XXX"; private final Log outgoingMessages = new Log(10000); - private final AtomicInteger msgSeqNum = new AtomicInteger(0); - private final AtomicInteger serverMsgSeqNum = new AtomicInteger(0); + private final AtomicInteger msgSeqNum; // = new AtomicInteger(0); + private final AtomicInteger serverMsgSeqNum; // = new AtomicInteger(0); private final AtomicInteger testReqID = new AtomicInteger(0); private final AtomicBoolean enabled = new AtomicBoolean(false); private final AtomicBoolean connStarted = new AtomicBoolean(false); private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); private final IHandlerContext context; private final InetSocketAddress address; + private final LocalTime resetTime; private Future heartbeatTimer = CompletableFuture.completedFuture(null); private Future testRequestTimer = CompletableFuture.completedFuture(null); @@ -141,6 +160,30 @@ public class FixHandler implements AutoCloseable, IHandler { public FixHandler(IHandlerContext context) { this.context = context; this.settings = (FixHandlerSettings) context.getSettings(); + if(settings.getStateFilePath() == null) { + msgSeqNum = new AtomicInteger(0); + serverMsgSeqNum = new AtomicInteger(0); + } else { + SequenceHolder sequences = Util.readSequences(settings.getStateFilePath()); + msgSeqNum = new AtomicInteger(sequences.getClientSeq()); + serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); + } + if(settings.getStartOfADayTime() != null) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss").withZone(ZoneOffset.UTC); + Instant instant = Instant.from(formatter.parse(settings.getStartOfADayTime())); + resetTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")).toLocalTime(); + LocalDateTime now = LocalDateTime.now(); + LocalDateTime scheduleTime; + if(now.with(resetTime).isAfter(now)) { + scheduleTime = now.with(resetTime); + } else { + scheduleTime = now.plusDays(1).with(resetTime); + } + long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); + executorService.schedule(this::reset, time, TimeUnit.MILLISECONDS); + } else { + resetTime = LocalTime.now(); + } String host = settings.getHost(); if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); int port = settings.getPort(); @@ -236,10 +279,25 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu return metadata; } - serverMsgSeqNum.incrementAndGet(); + FixField possDup = findField(message, POSS_DUP_TAG); + boolean isDup = false; + if(possDup != null) { + isDup = possDup.getValue().equals(IS_POSS_DUP); + } + int receivedMsgSeqNum = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); - if (serverMsgSeqNum.get() < receivedMsgSeqNum) { + if(receivedMsgSeqNum < serverMsgSeqNum.get() && !isDup) { + sendLogout(); + reconnectRequestTimer = executorService.schedule(this::sendLogon, settings.getReconnectDelay() + settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); + metadata.put(REJECT_REASON, "SeqNum is less than expected."); + if (LOGGER.isErrorEnabled()) LOGGER.error("Invalid message. SeqNum is less than expected {}: {}", serverMsgSeqNum.get(), message.toString(US_ASCII)); + return metadata; + } + + serverMsgSeqNum.incrementAndGet(); + + if (serverMsgSeqNum.get() < receivedMsgSeqNum && !isDup) { if (enabled.get()) { sendResendRequest(serverMsgSeqNum.get(), receivedMsgSeqNum); } @@ -257,6 +315,22 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu boolean connectionSuccessful = checkLogon(message); enabled.set(connectionSuccessful); if (connectionSuccessful) { + msgSeqNum.incrementAndGet(); + if(settings.isMaintainSessionBasedOnNextExpectedSeqNumber()) { + FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); + if(nextExpectedSeqField == null) { + metadata.put(REJECT_REASON, "No NextExpectedSeqNum field"); + if (LOGGER.isErrorEnabled()) LOGGER.error("Invalid message. No NextExpectedSeqNum in message: {}", message.toString(US_ASCII)); + return metadata; + } + + int nextExpectedSeqNumber = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); + if(serverMsgSeqNum.get() + 1 < nextExpectedSeqNumber) { + sendResendRequest(serverMsgSeqNum.get(), nextExpectedSeqNumber); + serverMsgSeqNum.set(nextExpectedSeqNumber); + } + } + if (!connStarted.get()){ connStarted.set(true); } @@ -318,10 +392,18 @@ private void resetSequence(ByteBuf message) { } } + private void reset() { + msgSeqNum.set(0); + serverMsgSeqNum.set(0); + LocalDateTime scheduleTime = LocalDateTime.now().plusDays(1).with(resetTime); + long diff = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); + executorService.schedule(this::reset, diff, TimeUnit.MILLISECONDS); + } + public void sendResendRequest(int beginSeqNo, int endSeqNo) { //do private lastSendTime = System.currentTimeMillis(); StringBuilder resendRequest = new StringBuilder(); - setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet()); + setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet(), false); resendRequest.append(BEGIN_SEQ_NO).append(beginSeqNo).append(SOH); resendRequest.append(END_SEQ_NO).append(endSeqNo).append(SOH); setChecksumAndBodyLength(resendRequest); @@ -331,7 +413,7 @@ public void sendResendRequest(int beginSeqNo, int endSeqNo) { //do private void sendResendRequest(int beginSeqNo) { //do private lastSendTime = System.currentTimeMillis(); StringBuilder resendRequest = new StringBuilder(); - setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet()); + setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet(), false); resendRequest.append(BEGIN_SEQ_NO).append(beginSeqNo); resendRequest.append(END_SEQ_NO).append(0); setChecksumAndBodyLength(resendRequest); @@ -365,11 +447,13 @@ private void handleResendRequest(ByteBuf message) { ByteBuf storedMsg = outgoingMessages.get(i); if (storedMsg == null) { StringBuilder heartbeat = new StringBuilder(); - setHeader(heartbeat, MSG_TYPE_HEARTBEAT, i); + setHeader(heartbeat, MSG_TYPE_HEARTBEAT, i, false); setChecksumAndBodyLength(heartbeat); channel.send(Unpooled.wrappedBuffer(heartbeat.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } else { if (LOGGER.isInfoEnabled()) LOGGER.info("Resending message: {}", storedMsg.toString(US_ASCII)); + FixField sendingTime = findField(storedMsg, SENDING_TIME_TAG); + sendingTime.insertNext(POSS_DUP_TAG, IS_POSS_DUP); channel.send(storedMsg, Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } } @@ -382,7 +466,7 @@ private void handleResendRequest(ByteBuf message) { private void sendSequenceReset() { lastSendTime = System.currentTimeMillis(); StringBuilder sequenceReset = new StringBuilder(); - setHeader(sequenceReset, MSG_TYPE_SEQUENCE_RESET, msgSeqNum.incrementAndGet()); + setHeader(sequenceReset, MSG_TYPE_SEQUENCE_RESET, msgSeqNum.incrementAndGet(), false); sequenceReset.append(NEW_SEQ_NO).append(msgSeqNum.get() + 1); setChecksumAndBodyLength(sequenceReset); @@ -464,9 +548,10 @@ public void onOutgoingUpdateTag(@NotNull ByteBuf message, @NotNull Map { + enabled.set(false); + try { + Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.incrementAndGet(), settings.getStateFilePath()); + } catch (IOException e) { + if (LOGGER.isErrorEnabled()) { + LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get()); + } + } + }, settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); + } + private String encrypt(String password) { return settings.getPasswordEncryptKeyFileType() .encrypt(Paths.get(settings.getPasswordEncryptKeyFilePath()), @@ -628,17 +740,10 @@ public void onClose(@NotNull IChannel channel) { @Override public void close() { - StringBuilder logout = new StringBuilder(); - setHeader(logout, MSG_TYPE_LOGOUT, msgSeqNum.incrementAndGet()); - setChecksumAndBodyLength(logout); - - if (enabled.get()) { - channel.send(Unpooled.wrappedBuffer(logout.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); - } - disconnectRequest = executorService.schedule(() -> enabled.set(false), settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); + sendLogout(); } - private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqNum) { + private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqNum, Boolean possDup) { stringBuilder.append(BEGIN_STRING_TAG).append("=").append(settings.getBeginString()); stringBuilder.append(MSG_TYPE).append(msgType); stringBuilder.append(MSG_SEQ_NUM).append(seqNum); @@ -646,6 +751,8 @@ private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqN if (settings.getTargetCompID() != null) stringBuilder.append(TARGET_COMP_ID).append(settings.getTargetCompID()); if (settings.getSenderSubID() != null) stringBuilder.append(SENDER_SUB_ID).append(settings.getSenderSubID()); stringBuilder.append(SENDING_TIME).append(getTime()); + if(possDup) + stringBuilder.append(POSS_DUP_TAG).append(IS_POSS_DUP); } private void setChecksumAndBodyLength(StringBuilder stringBuilder) { diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index 041c807..220b134 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel.Security; import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerSettings; +import java.time.LocalDateTime; + public class FixHandlerSettings implements IHandlerSettings { private String host = null; private int port = 0; @@ -36,6 +38,7 @@ public class FixHandlerSettings implements IHandlerSettings { private String newPassword; private String passwordEncryptKeyFilePath; private KeyFileType passwordEncryptKeyFileType = KeyFileType.PEM_PUBLIC_KEY; + private String stateFilePath; /** * Value from Java Security Standard Algorithm Names */ @@ -46,6 +49,9 @@ public class FixHandlerSettings implements IHandlerSettings { private String passwordEncryptAlgorithm = "RSA"; private Boolean resetSeqNumFlag = false; private Boolean resetOnLogon = false; + private Boolean maintainSessionBasedOnNextExpectedSeqNumber = false; + private Boolean isSaveAdminMessages = false; + private String startOfADayTime; private int testRequestDelay = 60; private int reconnectDelay = 5; private int disconnectRequestDelay = 5; @@ -198,6 +204,38 @@ public void setPasswordEncryptAlgorithm(String passwordEncryptAlgorithm) { this.passwordEncryptAlgorithm = passwordEncryptAlgorithm; } + public String getStateFilePath() { + return stateFilePath; + } + + public void setStateFilePath(String stateFilePath) { + this.stateFilePath = stateFilePath; + } + + public Boolean isMaintainSessionBasedOnNextExpectedSeqNumber() { + return maintainSessionBasedOnNextExpectedSeqNumber; + } + + public void setMaintainSessionBasedOnNextExpectedSeqNumber(Boolean maintainSessionBasedOnNextExpectedSeqNumber) { + this.maintainSessionBasedOnNextExpectedSeqNumber = maintainSessionBasedOnNextExpectedSeqNumber; + } + + public Boolean isSaveAdminMessages() { + return isSaveAdminMessages; + } + + public void setSaveAdminMessages(Boolean saveAdminMessages) { + isSaveAdminMessages = saveAdminMessages; + } + + public String getStartOfADayTime() { + return startOfADayTime; + } + + public void setStartOfADayTime(String startOfADayTime) { + this.startOfADayTime = startOfADayTime; + } + public void setResetSeqNumFlag(Boolean resetSeqNumFlag) { this.resetSeqNumFlag = resetSeqNumFlag; } public void setResetOnLogon(Boolean resetOnLogon) { this.resetOnLogon = resetOnLogon; } diff --git a/src/main/java/com/exactpro/th2/SequenceHolder.java b/src/main/java/com/exactpro/th2/SequenceHolder.java new file mode 100644 index 0000000..eeec1ae --- /dev/null +++ b/src/main/java/com/exactpro/th2/SequenceHolder.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2; + +public class SequenceHolder { + private final int clientSeq; + private final int serverSeq; + + public SequenceHolder(int clientSeq, int serverSeq) { + this.clientSeq = clientSeq; + this.serverSeq = serverSeq; + } + + public int getClientSeq() { + return clientSeq; + } + + public int getServerSeq() { + return serverSeq; + } +} diff --git a/src/main/java/com/exactpro/th2/constants/Constants.java b/src/main/java/com/exactpro/th2/constants/Constants.java index 8d6e0b4..588bc70 100644 --- a/src/main/java/com/exactpro/th2/constants/Constants.java +++ b/src/main/java/com/exactpro/th2/constants/Constants.java @@ -16,6 +16,10 @@ package com.exactpro.th2.constants; + +import java.util.Collections; +import java.util.Set; + public class Constants { public static final String SOH = "\001"; @@ -46,6 +50,8 @@ public class Constants { public static final Integer GAP_FILL_FLAG_TAG = 123; public static final Integer TEXT_TAG = 58; public static final Integer RESET_SEQ_NUM_TAG = 141; + public static final Integer NEXT_EXPECTED_SEQ_NUMBER_TAG = 789; + public static final Integer POSS_DUP_TAG = 43; //Fields public static final String BEGIN_STRING = SOH + BEGIN_STRING_TAG + "="; @@ -71,6 +77,8 @@ public class Constants { public static final String DEFAULT_APPL_VER_ID = SOH + DEFAULT_APPL_VER_ID_TAG + "="; public static final String SENDER_SUB_ID = SOH + SENDER_SUB_ID_TAG + "="; public static final String RESET_SEQ_NUM = SOH + RESET_SEQ_NUM_TAG + "="; + public static final String NEXT_EXPECTED_SEQ_NUM = SOH + NEXT_EXPECTED_SEQ_NUMBER_TAG + "="; + public static final String POSS_DUP = SOH + NEXT_EXPECTED_SEQ_NUMBER_TAG + "="; //message types public static final String MSG_TYPE_LOGON = "A"; @@ -80,4 +88,14 @@ public class Constants { public static final String MSG_TYPE_RESEND_REQUEST = "2"; public static final String MSG_TYPE_SEQUENCE_RESET = "4"; + public static final Set ADMIN_MESSAGES = Collections.unmodifiableSet( + Set.of( + MSG_TYPE_LOGON, MSG_TYPE_LOGOUT, + MSG_TYPE_HEARTBEAT, MSG_TYPE_RESEND_REQUEST, + MSG_TYPE_SEQUENCE_RESET, MSG_TYPE_TEST_REQUEST + ) + ); + + public static final String IS_POSS_DUP = "Y"; + } diff --git a/src/main/java/com/exactpro/th2/util/Util.java b/src/main/java/com/exactpro/th2/util/Util.java new file mode 100644 index 0000000..a16ca7e --- /dev/null +++ b/src/main/java/com/exactpro/th2/util/Util.java @@ -0,0 +1,43 @@ +/* + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.util; + +import com.exactpro.th2.SequenceHolder; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; + +public class Util { + public static SequenceHolder readSequences(String filePath) { + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + int firstLine = Integer.parseInt(br.readLine()); + int secondLine = Integer.parseInt(br.readLine()); + return new SequenceHolder(firstLine, secondLine); + } catch (IOException e) { + throw new IllegalStateException("Error while reading sequence file " + filePath, e); + } + } + + public static void writeSequences(int msgSeqNum, int serverSeqNum, String filePath) throws IOException { + BufferedWriter bw = new BufferedWriter(new FileWriter(filePath)); + bw.write(String.valueOf(msgSeqNum)); + bw.newLine(); + bw.write(String.valueOf(serverSeqNum)); + } +} diff --git a/src/test/java/com/exactpro/th2/FixHandlerTest.java b/src/test/java/com/exactpro/th2/FixHandlerTest.java index 3d02568..2403771 100644 --- a/src/test/java/com/exactpro/th2/FixHandlerTest.java +++ b/src/test/java/com/exactpro/th2/FixHandlerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2022 Exactpro (Exactpro Systems Limited) + * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -63,6 +64,8 @@ class FixHandlerTest { @BeforeAll static void init() { fixHandler.onOpen(channel); + ByteBuf logonResponse = Unpooled.wrappedBuffer("8=FIXT.1.1\0019=105\00135=A\00134=1\00149=server\00156=client\00150=system\00152=2014-12-22T10:15:30Z\00198=0\001108=30\0011137=9\0011409=0\00110=203\001".getBytes(StandardCharsets.US_ASCII)); + fixHandler.onIncoming(channel, logonResponse); oneMessageBuffer = Unpooled.wrappedBuffer("8=FIXT.1.1\0019=13\00135=AE\001552=1\00110=169\001".getBytes(StandardCharsets.US_ASCII)); buffer = Unpooled.wrappedBuffer(("8=FIXT.1.1\0019=13\00135=AE\001552=1\00110=169\0018=FIXT.1.1\0019=13\00135=NN" + "\001552=2\00110=100\0018=FIXT.1.1\0019=13\00135=NN\001552=2\00110=100\001").getBytes(StandardCharsets.US_ASCII)); @@ -132,12 +135,15 @@ void onReceiveCorrectMessagesTest() { void sendResendRequestTest() { String expectedLogon = "8=FIXT.1.1\u00019=105\u000135=A\u000134=2\u000149=client\u000156=server\u0001" + "50=trader\u000152=2014-12-22T10:15:30Z\u000198=0\u0001108=30\u00011137=9\u0001553=username\u0001554=pass\u000110=204\u0001"; // #1 sent logon + ByteBuf logonResponse = Unpooled.wrappedBuffer("8=FIXT.1.1\0019=105\00135=A\00134=2\00149=server\00156=client\00150=system\00152=2014-12-22T10:15:30Z\00198=0\001108=30\0011137=9\0011409=0\00110=203\001".getBytes(StandardCharsets.US_ASCII)); + // #2 sent resendRequest String expectedResendRequest = "8=FIXT.1.1\u00019=73\u000135=2\u000134=3\u000149=client\u000156=server" + // #2 sent resendRequest "\u000150=trader\u000152=2014-12-22T10:15:30Z\u00017=1\u000116=0\u000110=227\u0001"; channel.clearQueue(); fixHandler.sendLogon(); + fixHandler.onIncoming(channel, logonResponse); fixHandler.sendResendRequest(1); assertEquals(expectedLogon, new String(channel.getQueue().get(0).array())); //assertEquals(expectedHeartbeat, new String(client.getQueue().get(1).array())); @@ -148,12 +154,14 @@ void sendResendRequestTest() { void onConnectionTest() { channel.clearQueue(); fixHandler.onOpen(channel); + ByteBuf logonResponse = Unpooled.wrappedBuffer("8=FIXT.1.1\0019=105\00135=A\00134=1\00149=server\00156=client\00150=system\00152=2014-12-22T10:15:30Z\00198=0\001108=30\0011137=9\0011409=0\00110=203\001".getBytes(StandardCharsets.US_ASCII)); + fixHandler.onIncoming(channel, logonResponse); try { Thread.sleep(10000); } catch (InterruptedException e) { e.printStackTrace(); } - assertEquals("8=FIXT.1.1\u00019=105\u000135=A\u000134=7\u000149=client\u000156=server\u000150=trader\u000152=2014-12-22T10:15:30Z\u000198=0\u0001108=30\u00011137=9\u0001553=username\u0001554=pass\u000110=209\u0001", + assertEquals("8=FIXT.1.1\u00019=105\u000135=A\u000134=8\u000149=client\u000156=server\u000150=trader\u000152=2014-12-22T10:15:30Z\u000198=0\u0001108=30\u00011137=9\u0001553=username\u0001554=pass\u000110=210\u0001", new String(channel.getQueue().get(0).array())); } @@ -165,9 +173,9 @@ void onOutgoingMessageTest() { ByteBuf bufferForPrepareMessage4 = Unpooled.buffer().writeBytes("8=FIXT.1.1\0019=192\00135=A\00111=3428785\00122=8\00138=30\00140=2\00144=55\00148=INSTR1\00154=1\00159=0\00160=20220127-18:38:35\001526=11111\001528=A\001581=1\001453=4\001448=DEMO-CONN1\001447=D\001452=76\001448=0\001447=P\001452=3\001448=0\00147=P\001452=122\001448=3\001447=P\001452=12\00110=228\001".getBytes(StandardCharsets.US_ASCII)); String expectedMessage1 = "8=FIXT.1.1\u00019=70\u000135=A\u0001552=1\u000149=client\u000134=8\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u000110=132\u0001"; - String expectedMessage2 = "8=FIXT.1.1\u00019=65\u000134=4\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u0001552=1\u000110=157\u0001"; - String expectedMessage3 = "8=FIXT.1.1\u00019=243\u000135=A\u000134=5\u000149=client\u000156=server\u000150=trader\u000111=9977764\u000122=8\u000138=100\u000140=2\u000144=55\u000152=20220127-12:00:40.775\u000148=INSTR2\u000154=2\u000159=3\u000160=20220127-15:00:36\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN2\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u0001447=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=120\u0001"; - String expectedMessage4 = "8=FIXT.1.1\u00019=250\u000135=A\u000134=6\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u000111=3428785\u000122=8\u000138=30\u000140=2\u000144=55\u000148=INSTR1\u000154=1\u000159=0\u000160=20220127-18:38:35\u0001526=11111\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN1\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u000147=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=235\u0001"; + String expectedMessage2 = "8=FIXT.1.1\u00019=65\u000134=5\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u0001552=1\u000110=158\u0001"; + String expectedMessage3 = "8=FIXT.1.1\u00019=243\u000135=A\u000134=6\u000149=client\u000156=server\u000150=trader\u000111=9977764\u000122=8\u000138=100\u000140=2\u000144=55\u000152=20220127-12:00:40.775\u000148=INSTR2\u000154=2\u000159=3\u000160=20220127-15:00:36\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN2\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u0001447=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=121\u0001"; + String expectedMessage4 = "8=FIXT.1.1\u00019=250\u000135=A\u000134=7\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u000111=3428785\u000122=8\u000138=30\u000140=2\u000144=55\u000148=INSTR1\u000154=1\u000159=0\u000160=20220127-18:38:35\u0001526=11111\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN1\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u000147=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=236\u0001"; Map expected = new HashMap<>(); expected.put("MsgType", "A"); Map expected2 = new HashMap<>(); From 3adc36d2849265c650970aa214635c9b1c9521c2 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Wed, 18 Jan 2023 16:59:23 +0400 Subject: [PATCH 03/40] Rework logic in context: NextExpectedSeqNum is seq num we are expecting from server. --- .../java/com/exactpro/th2/FixHandler.java | 145 +++++++++--------- .../com/exactpro/th2/FixHandlerSettings.java | 41 +++-- .../java/com/exactpro/th2/SequenceHolder.java | 2 +- .../th2/util/LocalTimeDeserializer.java | 36 +++++ src/main/java/com/exactpro/th2/util/Util.java | 11 +- 5 files changed, 147 insertions(+), 88 deletions(-) create mode 100644 src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 120b86c..772b3fc 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -29,7 +29,6 @@ import io.netty.buffer.Unpooled; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.checkerframework.checker.units.qual.A; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,8 +37,6 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.time.Instant; import java.time.LocalDateTime; import java.time.LocalTime; @@ -50,7 +47,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.TimeZone; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -58,7 +54,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.findField; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.findLastField; @@ -66,9 +61,9 @@ import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.lastField; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.updateChecksum; import static com.exactpro.th2.conn.dirty.fix.FixByteBufUtilKt.updateLength; +import static com.exactpro.th2.conn.dirty.fix.KeyFileType.Companion.OperationMode.ENCRYPT_MODE; import static com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil.getEventId; import static com.exactpro.th2.conn.dirty.tcp.core.util.CommonUtil.toByteBuf; -import static com.exactpro.th2.conn.dirty.fix.KeyFileType.Companion.OperationMode.ENCRYPT_MODE; import static com.exactpro.th2.constants.Constants.ADMIN_MESSAGES; import static com.exactpro.th2.constants.Constants.BEGIN_SEQ_NO; import static com.exactpro.th2.constants.Constants.BEGIN_SEQ_NO_TAG; @@ -118,7 +113,6 @@ import static com.exactpro.th2.constants.Constants.TEXT_TAG; import static com.exactpro.th2.constants.Constants.USERNAME; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.indexOf; -import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.insert; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.isEmpty; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.set; import static com.exactpro.th2.util.MessageUtil.findByte; @@ -139,15 +133,14 @@ public class FixHandler implements AutoCloseable, IHandler { private static final String STUBBING_VALUE = "XXX"; private final Log outgoingMessages = new Log(10000); - private final AtomicInteger msgSeqNum; // = new AtomicInteger(0); - private final AtomicInteger serverMsgSeqNum; // = new AtomicInteger(0); + private final AtomicInteger msgSeqNum; + private final AtomicInteger serverMsgSeqNum; private final AtomicInteger testReqID = new AtomicInteger(0); private final AtomicBoolean enabled = new AtomicBoolean(false); private final AtomicBoolean connStarted = new AtomicBoolean(false); private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); private final IHandlerContext context; private final InetSocketAddress address; - private final LocalTime resetTime; private Future heartbeatTimer = CompletableFuture.completedFuture(null); private Future testRequestTimer = CompletableFuture.completedFuture(null); @@ -168,10 +161,8 @@ public FixHandler(IHandlerContext context) { msgSeqNum = new AtomicInteger(sequences.getClientSeq()); serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); } - if(settings.getStartOfADayTime() != null) { - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("HH:mm:ss").withZone(ZoneOffset.UTC); - Instant instant = Instant.from(formatter.parse(settings.getStartOfADayTime())); - resetTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")).toLocalTime(); + if(settings.getSessionStartTime() != null) { + LocalTime resetTime = settings.getSessionStartTime(); LocalDateTime now = LocalDateTime.now(); LocalDateTime scheduleTime; if(now.with(resetTime).isAfter(now)) { @@ -180,9 +171,20 @@ public FixHandler(IHandlerContext context) { scheduleTime = now.plusDays(1).with(resetTime); } long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); - executorService.schedule(this::reset, time, TimeUnit.MILLISECONDS); - } else { - resetTime = LocalTime.now(); + executorService.scheduleAtFixedRate(() -> reset(settings.getSessionEndTime() == null), time, 24, TimeUnit.HOURS); + } + + if(settings.getSessionEndTime() != null) { + LocalTime resetTime = settings.getSessionEndTime(); + LocalDateTime now = LocalDateTime.now(); + LocalDateTime scheduleTime; + if(now.with(resetTime).isAfter(now)) { + scheduleTime = now.with(resetTime); + } else { + scheduleTime = now.plusDays(1).with(resetTime); + } + long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); + executorService.scheduleAtFixedRate(() -> enabled.set(true), time, 24, TimeUnit.HOURS); } String host = settings.getHost(); if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); @@ -297,10 +299,8 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu serverMsgSeqNum.incrementAndGet(); - if (serverMsgSeqNum.get() < receivedMsgSeqNum && !isDup) { - if (enabled.get()) { - sendResendRequest(serverMsgSeqNum.get(), receivedMsgSeqNum); - } + if (serverMsgSeqNum.get() < receivedMsgSeqNum && !isDup && enabled.get()) { + sendResendRequest(serverMsgSeqNum.get(), receivedMsgSeqNum); } String msgTypeValue = requireNonNull(msgType.getValue()); @@ -313,10 +313,9 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu case MSG_TYPE_LOGON: if (LOGGER.isInfoEnabled()) LOGGER.info("Logon received - {}", message.toString(US_ASCII)); boolean connectionSuccessful = checkLogon(message); - enabled.set(connectionSuccessful); if (connectionSuccessful) { msgSeqNum.incrementAndGet(); - if(settings.isMaintainSessionBasedOnNextExpectedSeqNumber()) { + if(settings.useNextExpectedSeqNum()) { FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); if(nextExpectedSeqField == null) { metadata.put(REJECT_REASON, "No NextExpectedSeqNum field"); @@ -324,13 +323,15 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu return metadata; } - int nextExpectedSeqNumber = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); - if(serverMsgSeqNum.get() + 1 < nextExpectedSeqNumber) { - sendResendRequest(serverMsgSeqNum.get(), nextExpectedSeqNumber); - serverMsgSeqNum.set(nextExpectedSeqNumber); + int nextExpectedSeqNumber = Integer.parseInt(requireNonNull(nextExpectedSeqField.getValue())); + int seqNum = msgSeqNum.get(); + if(nextExpectedSeqNumber < seqNum) { + recovery(nextExpectedSeqNumber, seqNum); } } + enabled.set(true); + if (!connStarted.get()){ connStarted.set(true); } @@ -342,6 +343,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu testRequestTimer = executorService.schedule(this::sendTestRequest, settings.getTestRequestDelay(), TimeUnit.SECONDS); } else { + enabled.set(false); reconnectRequestTimer = executorService.schedule(this::sendLogon, settings.getReconnectDelay(), TimeUnit.SECONDS); } break; @@ -392,18 +394,16 @@ private void resetSequence(ByteBuf message) { } } - private void reset() { + private void reset(Boolean disableClient) { msgSeqNum.set(0); serverMsgSeqNum.set(0); - LocalDateTime scheduleTime = LocalDateTime.now().plusDays(1).with(resetTime); - long diff = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); - executorService.schedule(this::reset, diff, TimeUnit.MILLISECONDS); + if(disableClient) enabled.set(false); } public void sendResendRequest(int beginSeqNo, int endSeqNo) { //do private lastSendTime = System.currentTimeMillis(); StringBuilder resendRequest = new StringBuilder(); - setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet(), false); + setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet()); resendRequest.append(BEGIN_SEQ_NO).append(beginSeqNo).append(SOH); resendRequest.append(END_SEQ_NO).append(endSeqNo).append(SOH); setChecksumAndBodyLength(resendRequest); @@ -413,7 +413,7 @@ public void sendResendRequest(int beginSeqNo, int endSeqNo) { //do private void sendResendRequest(int beginSeqNo) { //do private lastSendTime = System.currentTimeMillis(); StringBuilder resendRequest = new StringBuilder(); - setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet(), false); + setHeader(resendRequest, MSG_TYPE_RESEND_REQUEST, msgSeqNum.incrementAndGet()); resendRequest.append(BEGIN_SEQ_NO).append(beginSeqNo); resendRequest.append(END_SEQ_NO).append(0); setChecksumAndBodyLength(resendRequest); @@ -439,34 +439,38 @@ private void handleResendRequest(ByteBuf message) { try { // FIXME: there is not syn on the outgoing sequence. Should make operations with seq more careful - if (endSeqNo == 0) { - endSeqNo = msgSeqNum.get(); - } - LOGGER.info("Returning messages from {} to {}", beginSeqNo, endSeqNo); - for (int i = beginSeqNo; i <= endSeqNo; i++) { - ByteBuf storedMsg = outgoingMessages.get(i); - if (storedMsg == null) { - StringBuilder heartbeat = new StringBuilder(); - setHeader(heartbeat, MSG_TYPE_HEARTBEAT, i, false); - setChecksumAndBodyLength(heartbeat); - channel.send(Unpooled.wrappedBuffer(heartbeat.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); - } else { - if (LOGGER.isInfoEnabled()) LOGGER.info("Resending message: {}", storedMsg.toString(US_ASCII)); - FixField sendingTime = findField(storedMsg, SENDING_TIME_TAG); - sendingTime.insertNext(POSS_DUP_TAG, IS_POSS_DUP); - channel.send(storedMsg, Collections.emptyMap(), null, IChannel.SendMode.MANGLE); - } - } + recovery(beginSeqNo, endSeqNo); } catch (Exception e) { sendSequenceReset(); } } } + private void recovery(int beginSeqNo, int endSeqNo) { + if (endSeqNo == 0) { + endSeqNo = msgSeqNum.get(); + } + LOGGER.info("Returning messages from {} to {}", beginSeqNo, endSeqNo); + for (int i = beginSeqNo; i <= endSeqNo; i++) { + ByteBuf storedMsg = outgoingMessages.get(i); + if (storedMsg == null) { + StringBuilder heartbeat = new StringBuilder(); + setHeader(heartbeat, MSG_TYPE_HEARTBEAT, i); + setChecksumAndBodyLength(heartbeat); + channel.send(Unpooled.wrappedBuffer(heartbeat.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, SendMode.MANGLE); + } else { + if (LOGGER.isInfoEnabled()) LOGGER.info("Resending message: {}", storedMsg.toString(US_ASCII)); + FixField sendingTime = findField(storedMsg, SENDING_TIME_TAG); + sendingTime.insertNext(POSS_DUP_TAG, IS_POSS_DUP); + channel.send(storedMsg, Collections.emptyMap(), null, SendMode.MANGLE); + } + } + } + private void sendSequenceReset() { lastSendTime = System.currentTimeMillis(); StringBuilder sequenceReset = new StringBuilder(); - setHeader(sequenceReset, MSG_TYPE_SEQUENCE_RESET, msgSeqNum.incrementAndGet(), false); + setHeader(sequenceReset, MSG_TYPE_SEQUENCE_RESET, msgSeqNum.incrementAndGet()); sequenceReset.append(NEW_SEQ_NO).append(msgSeqNum.get() + 1); setChecksumAndBodyLength(sequenceReset); @@ -548,10 +552,9 @@ public void onOutgoingUpdateTag(@NotNull ByteBuf message, @NotNull Map { enabled.set(false); - try { - Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.incrementAndGet(), settings.getStateFilePath()); - } catch (IOException e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get()); - } - } }, settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); } @@ -743,7 +748,7 @@ public void close() { sendLogout(); } - private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqNum, Boolean possDup) { + private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqNum) { stringBuilder.append(BEGIN_STRING_TAG).append("=").append(settings.getBeginString()); stringBuilder.append(MSG_TYPE).append(msgType); stringBuilder.append(MSG_SEQ_NUM).append(seqNum); @@ -751,8 +756,6 @@ private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqN if (settings.getTargetCompID() != null) stringBuilder.append(TARGET_COMP_ID).append(settings.getTargetCompID()); if (settings.getSenderSubID() != null) stringBuilder.append(SENDER_SUB_ID).append(settings.getSenderSubID()); stringBuilder.append(SENDING_TIME).append(getTime()); - if(possDup) - stringBuilder.append(POSS_DUP_TAG).append(IS_POSS_DUP); } private void setChecksumAndBodyLength(StringBuilder stringBuilder) { diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index 220b134..d8330c5 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -19,8 +19,11 @@ import com.exactpro.th2.conn.dirty.fix.KeyFileType; import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel.Security; import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerSettings; +import com.exactpro.th2.util.LocalTimeDeserializer; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import java.time.LocalDateTime; +import java.time.LocalTime; public class FixHandlerSettings implements IHandlerSettings { private String host = null; @@ -49,9 +52,17 @@ public class FixHandlerSettings implements IHandlerSettings { private String passwordEncryptAlgorithm = "RSA"; private Boolean resetSeqNumFlag = false; private Boolean resetOnLogon = false; - private Boolean maintainSessionBasedOnNextExpectedSeqNumber = false; + private Boolean useNextExpectedSeqNum = false; private Boolean isSaveAdminMessages = false; - private String startOfADayTime; + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "HH:mm:ss.SSS") + @JsonDeserialize(using = LocalTimeDeserializer.class) + private LocalTime sessionStartTime; + + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "HH:mm:ss.SSS") + @JsonDeserialize(using = LocalTimeDeserializer.class) + private LocalTime sessionEndTime; + private int testRequestDelay = 60; private int reconnectDelay = 5; private int disconnectRequestDelay = 5; @@ -212,12 +223,12 @@ public void setStateFilePath(String stateFilePath) { this.stateFilePath = stateFilePath; } - public Boolean isMaintainSessionBasedOnNextExpectedSeqNumber() { - return maintainSessionBasedOnNextExpectedSeqNumber; + public Boolean useNextExpectedSeqNum() { + return useNextExpectedSeqNum; } - public void setMaintainSessionBasedOnNextExpectedSeqNumber(Boolean maintainSessionBasedOnNextExpectedSeqNumber) { - this.maintainSessionBasedOnNextExpectedSeqNumber = maintainSessionBasedOnNextExpectedSeqNumber; + public void setUseNextExpectedSeqNum(Boolean useNextExpectedSeqNum) { + this.useNextExpectedSeqNum = useNextExpectedSeqNum; } public Boolean isSaveAdminMessages() { @@ -228,12 +239,20 @@ public void setSaveAdminMessages(Boolean saveAdminMessages) { isSaveAdminMessages = saveAdminMessages; } - public String getStartOfADayTime() { - return startOfADayTime; + public LocalTime getSessionStartTime() { + return sessionStartTime; + } + + public void setSessionStartTime(LocalTime sessionStartTime) { + this.sessionStartTime = sessionStartTime; + } + + public LocalTime getSessionEndTime() { + return sessionEndTime; } - public void setStartOfADayTime(String startOfADayTime) { - this.startOfADayTime = startOfADayTime; + public void setSessionEndTime(LocalTime sessionEndTime) { + this.sessionEndTime = sessionEndTime; } public void setResetSeqNumFlag(Boolean resetSeqNumFlag) { this.resetSeqNumFlag = resetSeqNumFlag; } diff --git a/src/main/java/com/exactpro/th2/SequenceHolder.java b/src/main/java/com/exactpro/th2/SequenceHolder.java index eeec1ae..6fbfc05 100644 --- a/src/main/java/com/exactpro/th2/SequenceHolder.java +++ b/src/main/java/com/exactpro/th2/SequenceHolder.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java b/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java new file mode 100644 index 0000000..7e94932 --- /dev/null +++ b/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.util; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.time.LocalTime; +import java.time.ZoneOffset; + +public class LocalTimeDeserializer extends StdDeserializer { + + public LocalTimeDeserializer() { + super(LocalTime.class); + } + + @Override + public LocalTime deserialize(JsonParser parser, DeserializationContext context) throws IOException { + return LocalTime.parse(parser.getValueAsString()).atOffset(ZoneOffset.UTC).toLocalTime(); + } +} diff --git a/src/main/java/com/exactpro/th2/util/Util.java b/src/main/java/com/exactpro/th2/util/Util.java index a16ca7e..0f6ea24 100644 --- a/src/main/java/com/exactpro/th2/util/Util.java +++ b/src/main/java/com/exactpro/th2/util/Util.java @@ -1,5 +1,5 @@ /* - * Copyright 2022-2023 Exactpro (Exactpro Systems Limited) + * Copyright 2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,9 +35,10 @@ public static SequenceHolder readSequences(String filePath) { } public static void writeSequences(int msgSeqNum, int serverSeqNum, String filePath) throws IOException { - BufferedWriter bw = new BufferedWriter(new FileWriter(filePath)); - bw.write(String.valueOf(msgSeqNum)); - bw.newLine(); - bw.write(String.valueOf(serverSeqNum)); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(filePath))) { + bw.write(String.valueOf(msgSeqNum)); + bw.newLine(); + bw.write(String.valueOf(serverSeqNum)); + } } } From 8f6bf06c2da5dc340063909d6429905892c70b46 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Wed, 18 Jan 2023 17:27:13 +0400 Subject: [PATCH 04/40] fix tests --- src/main/java/com/exactpro/th2/FixHandler.java | 16 +++++++++------- .../java/com/exactpro/th2/FixHandlerTest.java | 8 ++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 772b3fc..78c58ca 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -314,7 +314,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if (LOGGER.isInfoEnabled()) LOGGER.info("Logon received - {}", message.toString(US_ASCII)); boolean connectionSuccessful = checkLogon(message); if (connectionSuccessful) { - msgSeqNum.incrementAndGet(); + if(settings.useNextExpectedSeqNum()) { FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); if(nextExpectedSeqField == null) { @@ -676,7 +676,7 @@ public void sendLogon() { else reset = settings.getResetOnLogon(); if (reset) msgSeqNum.getAndSet(0); - setHeader(logon, MSG_TYPE_LOGON, msgSeqNum.get() + 1); + setHeader(logon, MSG_TYPE_LOGON, msgSeqNum.incrementAndGet()); if (settings.useNextExpectedSeqNum()) logon.append(NEXT_EXPECTED_SEQ_NUM).append(serverMsgSeqNum.get() + 1); if (settings.getEncryptMethod() != null) logon.append(ENCRYPT_METHOD).append(settings.getEncryptMethod()); logon.append(HEART_BT_INT).append(settings.getHeartBtInt()); @@ -711,11 +711,13 @@ private void sendLogout() { if (enabled.get()) { channel.send(Unpooled.wrappedBuffer(logout.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } - try { - Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.incrementAndGet(), settings.getStateFilePath()); - } catch (IOException e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get()); + if(settings.getStateFilePath() != null) { + try { + Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.incrementAndGet(), settings.getStateFilePath()); + } catch (IOException e) { + if (LOGGER.isErrorEnabled()) { + LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get()); + } } } disconnectRequest = executorService.schedule(() -> { diff --git a/src/test/java/com/exactpro/th2/FixHandlerTest.java b/src/test/java/com/exactpro/th2/FixHandlerTest.java index 2403771..255f51b 100644 --- a/src/test/java/com/exactpro/th2/FixHandlerTest.java +++ b/src/test/java/com/exactpro/th2/FixHandlerTest.java @@ -161,7 +161,7 @@ void onConnectionTest() { } catch (InterruptedException e) { e.printStackTrace(); } - assertEquals("8=FIXT.1.1\u00019=105\u000135=A\u000134=8\u000149=client\u000156=server\u000150=trader\u000152=2014-12-22T10:15:30Z\u000198=0\u0001108=30\u00011137=9\u0001553=username\u0001554=pass\u000110=210\u0001", + assertEquals("8=FIXT.1.1\u00019=105\u000135=A\u000134=7\u000149=client\u000156=server\u000150=trader\u000152=2014-12-22T10:15:30Z\u000198=0\u0001108=30\u00011137=9\u0001553=username\u0001554=pass\u000110=209\u0001", new String(channel.getQueue().get(0).array())); } @@ -173,9 +173,9 @@ void onOutgoingMessageTest() { ByteBuf bufferForPrepareMessage4 = Unpooled.buffer().writeBytes("8=FIXT.1.1\0019=192\00135=A\00111=3428785\00122=8\00138=30\00140=2\00144=55\00148=INSTR1\00154=1\00159=0\00160=20220127-18:38:35\001526=11111\001528=A\001581=1\001453=4\001448=DEMO-CONN1\001447=D\001452=76\001448=0\001447=P\001452=3\001448=0\00147=P\001452=122\001448=3\001447=P\001452=12\00110=228\001".getBytes(StandardCharsets.US_ASCII)); String expectedMessage1 = "8=FIXT.1.1\u00019=70\u000135=A\u0001552=1\u000149=client\u000134=8\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u000110=132\u0001"; - String expectedMessage2 = "8=FIXT.1.1\u00019=65\u000134=5\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u0001552=1\u000110=158\u0001"; - String expectedMessage3 = "8=FIXT.1.1\u00019=243\u000135=A\u000134=6\u000149=client\u000156=server\u000150=trader\u000111=9977764\u000122=8\u000138=100\u000140=2\u000144=55\u000152=20220127-12:00:40.775\u000148=INSTR2\u000154=2\u000159=3\u000160=20220127-15:00:36\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN2\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u0001447=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=121\u0001"; - String expectedMessage4 = "8=FIXT.1.1\u00019=250\u000135=A\u000134=7\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u000111=3428785\u000122=8\u000138=30\u000140=2\u000144=55\u000148=INSTR1\u000154=1\u000159=0\u000160=20220127-18:38:35\u0001526=11111\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN1\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u000147=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=236\u0001"; + String expectedMessage2 = "8=FIXT.1.1\u00019=65\u000134=4\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u0001552=1\u000110=157\u0001"; + String expectedMessage3 = "8=FIXT.1.1\u00019=243\u000135=A\u000134=5\u000149=client\u000156=server\u000150=trader\u000111=9977764\u000122=8\u000138=100\u000140=2\u000144=55\u000152=20220127-12:00:40.775\u000148=INSTR2\u000154=2\u000159=3\u000160=20220127-15:00:36\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN2\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u0001447=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=120\u0001"; + String expectedMessage4 = "8=FIXT.1.1\u00019=250\u000135=A\u000134=6\u000149=client\u000156=server\u000152=2014-12-22T10:15:30Z\u000150=trader\u000111=3428785\u000122=8\u000138=30\u000140=2\u000144=55\u000148=INSTR1\u000154=1\u000159=0\u000160=20220127-18:38:35\u0001526=11111\u0001528=A\u0001581=1\u0001453=4\u0001448=DEMO-CONN1\u0001447=D\u0001452=76\u0001448=0\u0001447=P\u0001452=3\u0001448=0\u000147=P\u0001452=122\u0001448=3\u0001447=P\u0001452=12\u000110=235\u0001"; Map expected = new HashMap<>(); expected.put("MsgType", "A"); Map expected2 = new HashMap<>(); From 901d3af362656553f77f4051721a6ca59eb5fb97 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Wed, 18 Jan 2023 17:45:41 +0400 Subject: [PATCH 05/40] fix start and end session handling logic --- src/main/java/com/exactpro/th2/FixHandler.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 78c58ca..487375b 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -171,7 +171,7 @@ public FixHandler(IHandlerContext context) { scheduleTime = now.plusDays(1).with(resetTime); } long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); - executorService.scheduleAtFixedRate(() -> reset(settings.getSessionEndTime() == null), time, 24, TimeUnit.HOURS); + executorService.scheduleAtFixedRate(this::reset, time, 24, TimeUnit.HOURS); } if(settings.getSessionEndTime() != null) { @@ -184,7 +184,7 @@ public FixHandler(IHandlerContext context) { scheduleTime = now.plusDays(1).with(resetTime); } long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); - executorService.scheduleAtFixedRate(() -> enabled.set(true), time, 24, TimeUnit.HOURS); + executorService.scheduleAtFixedRate(this::close, time, 24, TimeUnit.HOURS); } String host = settings.getHost(); if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); @@ -394,10 +394,10 @@ private void resetSequence(ByteBuf message) { } } - private void reset(Boolean disableClient) { + private void reset() { msgSeqNum.set(0); serverMsgSeqNum.set(0); - if(disableClient) enabled.set(false); + sendLogon(); } public void sendResendRequest(int beginSeqNo, int endSeqNo) { //do private From 81ae256dc525303a67b9019768287ba2cfc31320 Mon Sep 17 00:00:00 2001 From: "ivan.druzhinin" Date: Wed, 18 Jan 2023 18:36:48 +0400 Subject: [PATCH 06/40] SendingTime with nanosecond precision --- src/main/java/com/exactpro/th2/FixHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index da58093..bcd20c8 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -668,7 +668,7 @@ public int getBodyLength(ByteBuf message) { } public String getTime() { - String FIX_DATE_TIME_FORMAT_MS = "yyyyMMdd-HH:mm:ss.SSS"; + String FIX_DATE_TIME_FORMAT_MS = "yyyyMMdd-HH:mm:ss.SSSSSSSSS"; LocalDateTime datetime = LocalDateTime.now(); return DateTimeFormatter.ofPattern(FIX_DATE_TIME_FORMAT_MS).format(datetime); } From 8584bc5be7fd98b368d6549ef415a889d8a3438e Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Wed, 18 Jan 2023 20:12:00 +0400 Subject: [PATCH 07/40] UTC based time calculations --- .../java/com/exactpro/th2/FixHandler.java | 31 ++++++++----------- .../com/exactpro/th2/FixHandlerSettings.java | 2 -- .../th2/util/LocalTimeDeserializer.java | 4 ++- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 487375b..59fa7be 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -42,7 +42,10 @@ import java.time.LocalTime; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -145,7 +148,6 @@ public class FixHandler implements AutoCloseable, IHandler { private Future heartbeatTimer = CompletableFuture.completedFuture(null); private Future testRequestTimer = CompletableFuture.completedFuture(null); private Future reconnectRequestTimer = CompletableFuture.completedFuture(null); - private Future disconnectRequest = CompletableFuture.completedFuture(null); private volatile IChannel channel; protected FixHandlerSettings settings; private long lastSendTime = System.currentTimeMillis(); @@ -162,28 +164,29 @@ public FixHandler(IHandlerContext context) { serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); } if(settings.getSessionStartTime() != null) { + Objects.requireNonNull(settings.getSessionEndTime(), "Session end is required when session start is presented"); LocalTime resetTime = settings.getSessionStartTime(); - LocalDateTime now = LocalDateTime.now(); - LocalDateTime scheduleTime; + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime scheduleTime; if(now.with(resetTime).isAfter(now)) { scheduleTime = now.with(resetTime); } else { scheduleTime = now.plusDays(1).with(resetTime); } - long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); + long time = now.until(scheduleTime, ChronoUnit.HOURS); executorService.scheduleAtFixedRate(this::reset, time, 24, TimeUnit.HOURS); } if(settings.getSessionEndTime() != null) { LocalTime resetTime = settings.getSessionEndTime(); - LocalDateTime now = LocalDateTime.now(); - LocalDateTime scheduleTime; + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime scheduleTime; if(now.with(resetTime).isAfter(now)) { scheduleTime = now.with(resetTime); } else { scheduleTime = now.plusDays(1).with(resetTime); } - long time = scheduleTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() - System.currentTimeMillis(); + long time = now.until(scheduleTime, ChronoUnit.HOURS); executorService.scheduleAtFixedRate(this::close, time, 24, TimeUnit.HOURS); } String host = settings.getHost(); @@ -314,7 +317,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if (LOGGER.isInfoEnabled()) LOGGER.info("Logon received - {}", message.toString(US_ASCII)); boolean connectionSuccessful = checkLogon(message); if (connectionSuccessful) { - + msgSeqNum.incrementAndGet(); if(settings.useNextExpectedSeqNum()) { FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); if(nextExpectedSeqField == null) { @@ -358,9 +361,6 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu serverMsgSeqNum.getAndSet(Integer.parseInt(msgSeqNumValue.getValue())); } } - if (disconnectRequest != null && !disconnectRequest.isCancelled()) { - disconnectRequest.cancel(false); - } enabled.set(false); context.send(CommonUtil.toEvent("logout for sender - " + settings.getSenderCompID()));//make more useful break; @@ -426,9 +426,6 @@ void sendResendRequest(int beginSeqNo) { //do private } private void handleResendRequest(ByteBuf message) { - if (disconnectRequest != null && !disconnectRequest.isCancelled()) { - disconnectRequest.cancel(false); - } FixField strBeginSeqNo = findField(message, BEGIN_SEQ_NO_TAG); FixField strEndSeqNo = findField(message, END_SEQ_NO_TAG); @@ -676,7 +673,7 @@ public void sendLogon() { else reset = settings.getResetOnLogon(); if (reset) msgSeqNum.getAndSet(0); - setHeader(logon, MSG_TYPE_LOGON, msgSeqNum.incrementAndGet()); + setHeader(logon, MSG_TYPE_LOGON, msgSeqNum.get() + 1); if (settings.useNextExpectedSeqNum()) logon.append(NEXT_EXPECTED_SEQ_NUM).append(serverMsgSeqNum.get() + 1); if (settings.getEncryptMethod() != null) logon.append(ENCRYPT_METHOD).append(settings.getEncryptMethod()); logon.append(HEART_BT_INT).append(settings.getHeartBtInt()); @@ -720,9 +717,7 @@ private void sendLogout() { } } } - disconnectRequest = executorService.schedule(() -> { - enabled.set(false); - }, settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); + enabled.set(false); } private String encrypt(String password) { diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index d8330c5..f93be89 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -55,11 +55,9 @@ public class FixHandlerSettings implements IHandlerSettings { private Boolean useNextExpectedSeqNum = false; private Boolean isSaveAdminMessages = false; - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "HH:mm:ss.SSS") @JsonDeserialize(using = LocalTimeDeserializer.class) private LocalTime sessionStartTime; - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "HH:mm:ss.SSS") @JsonDeserialize(using = LocalTimeDeserializer.class) private LocalTime sessionEndTime; diff --git a/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java b/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java index 7e94932..9e2376a 100644 --- a/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java +++ b/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.time.LocalTime; import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; public class LocalTimeDeserializer extends StdDeserializer { @@ -31,6 +33,6 @@ public LocalTimeDeserializer() { @Override public LocalTime deserialize(JsonParser parser, DeserializationContext context) throws IOException { - return LocalTime.parse(parser.getValueAsString()).atOffset(ZoneOffset.UTC).toLocalTime(); + return LocalTime.parse(parser.getValueAsString(), DateTimeFormatter.ISO_TIME); } } From f7f2d5d23b512f02b4f4a76dea67113c28562202 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Thu, 19 Jan 2023 19:27:36 +0400 Subject: [PATCH 08/40] allow empty state file --- src/main/java/com/exactpro/th2/FixHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 59fa7be..ea3e5d8 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -33,6 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; @@ -155,7 +156,7 @@ public class FixHandler implements AutoCloseable, IHandler { public FixHandler(IHandlerContext context) { this.context = context; this.settings = (FixHandlerSettings) context.getSettings(); - if(settings.getStateFilePath() == null) { + if(settings.getStateFilePath() == null || !new File(settings.getStateFilePath()).exists()) { msgSeqNum = new AtomicInteger(0); serverMsgSeqNum = new AtomicInteger(0); } else { From 874ff57c251d45497b9354865388be368ba9bf9f Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Thu, 19 Jan 2023 20:26:08 +0400 Subject: [PATCH 09/40] logout only when connection is enabled --- src/main/java/com/exactpro/th2/FixHandler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index ea3e5d8..e222ecd 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -711,7 +711,7 @@ private void sendLogout() { } if(settings.getStateFilePath() != null) { try { - Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.incrementAndGet(), settings.getStateFilePath()); + Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.get() + 1, settings.getStateFilePath()); } catch (IOException e) { if (LOGGER.isErrorEnabled()) { LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get()); @@ -743,7 +743,9 @@ public void onClose(@NotNull IChannel channel) { @Override public void close() { - sendLogout(); + if(enabled.get()) { + sendLogout(); + } } private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqNum) { From 27e4fd053ebe7019a4ddaa637c714d6666ae831c Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Thu, 19 Jan 2023 20:41:58 +0400 Subject: [PATCH 10/40] Fix logic for logout with 'Expecting {x} but received {y}' --- src/main/java/com/exactpro/th2/FixHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index e222ecd..a27b8e3 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -358,7 +358,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu LOGGER.warn("Received Logout has text (58) tag: {}", text.getValue()); String value = StringUtils.substringBetween(text.getValue(), "expecting ", " but received"); if (value != null) { - msgSeqNum.getAndSet(Integer.parseInt(value)-2); + msgSeqNum.getAndSet(Integer.parseInt(value)-1); serverMsgSeqNum.getAndSet(Integer.parseInt(msgSeqNumValue.getValue())); } } From 2d829009f12a9877b4b2ddadce63185ad6406a66 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Thu, 19 Jan 2023 21:53:19 +0400 Subject: [PATCH 11/40] Open channel on start --- build.gradle | 2 +- src/main/java/com/exactpro/th2/FixHandler.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9867438..7653e1d 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ dependencies { implementation 'com.exactpro.th2:common:3.44.0' implementation 'com.exactpro.th2:netty-bytebuf-utils:0.0.1' - implementation 'com.exactpro.th2:conn-dirty-tcp-core:2.0.3' + implementation 'com.exactpro.th2:conn-dirty-tcp-core:2.0.4' implementation 'org.slf4j:slf4j-api:1.7.33' implementation 'io.github.microutils:kotlin-logging:2.1.23' diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index a27b8e3..6c11e41 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -207,6 +207,7 @@ public FixHandler(IHandlerContext context) { @Override public void onStart() { channel = context.createChannel(address, settings.getSecurity(), Map.of(), true, settings.getReconnectDelay() * 1000L, Integer.MAX_VALUE); + channel.open(); } @NotNull From 7077e4afaedda80fdc1b19f3e9249b4160184bd4 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Thu, 19 Jan 2023 21:56:50 +0400 Subject: [PATCH 12/40] return logic for expecting x, but received y --- src/main/java/com/exactpro/th2/FixHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 6c11e41..9eb2807 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -359,7 +359,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu LOGGER.warn("Received Logout has text (58) tag: {}", text.getValue()); String value = StringUtils.substringBetween(text.getValue(), "expecting ", " but received"); if (value != null) { - msgSeqNum.getAndSet(Integer.parseInt(value)-1); + msgSeqNum.getAndSet(Integer.parseInt(value) - 2); serverMsgSeqNum.getAndSet(Integer.parseInt(msgSeqNumValue.getValue())); } } From d835a06e5809fbc919a7d1db75577a6d58b0e8c9 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Thu, 19 Jan 2023 22:34:35 +0400 Subject: [PATCH 13/40] Fix sequence update logic --- build.gradle | 8 ++++++++ src/main/java/com/exactpro/th2/FixHandler.java | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 7653e1d..d0efdbb 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,15 @@ version release_version sourceCompatibility = 11 targetCompatibility = 11 +ext { + sharedDir = file("${project.rootDir}/shared") +} + repositories { + maven { + name 'MavenLocal' + url sharedDir + } mavenCentral() maven { diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 9eb2807..179aa64 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -142,6 +142,7 @@ public class FixHandler implements AutoCloseable, IHandler { private final AtomicInteger testReqID = new AtomicInteger(0); private final AtomicBoolean enabled = new AtomicBoolean(false); private final AtomicBoolean connStarted = new AtomicBoolean(false); + private final AtomicBoolean logonResponded = new AtomicBoolean(true); private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); private final IHandlerContext context; private final InetSocketAddress address; @@ -292,11 +293,16 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu isDup = possDup.getValue().equals(IS_POSS_DUP); } + if(!logonResponded.get()) { + msgSeqNum.incrementAndGet(); + logonResponded.set(true); + } + int receivedMsgSeqNum = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); if(receivedMsgSeqNum < serverMsgSeqNum.get() && !isDup) { sendLogout(); - reconnectRequestTimer = executorService.schedule(this::sendLogon, settings.getReconnectDelay() + settings.getDisconnectRequestDelay(), TimeUnit.SECONDS); + reconnectRequestTimer = executorService.schedule(this::sendLogon, settings.getReconnectDelay(), TimeUnit.SECONDS); metadata.put(REJECT_REASON, "SeqNum is less than expected."); if (LOGGER.isErrorEnabled()) LOGGER.error("Invalid message. SeqNum is less than expected {}: {}", serverMsgSeqNum.get(), message.toString(US_ASCII)); return metadata; @@ -319,7 +325,6 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if (LOGGER.isInfoEnabled()) LOGGER.info("Logon received - {}", message.toString(US_ASCII)); boolean connectionSuccessful = checkLogon(message); if (connectionSuccessful) { - msgSeqNum.incrementAndGet(); if(settings.useNextExpectedSeqNum()) { FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); if(nextExpectedSeqField == null) { @@ -359,7 +364,13 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu LOGGER.warn("Received Logout has text (58) tag: {}", text.getValue()); String value = StringUtils.substringBetween(text.getValue(), "expecting ", " but received"); if (value != null) { - msgSeqNum.getAndSet(Integer.parseInt(value) - 2); + if (heartbeatTimer != null) { + heartbeatTimer.cancel(false); + } + if (testRequestTimer != null) { + testRequestTimer.cancel(false); + } + msgSeqNum.getAndSet(Integer.parseInt(value) - 1); serverMsgSeqNum.getAndSet(Integer.parseInt(msgSeqNumValue.getValue())); } } @@ -699,6 +710,7 @@ public void sendLogon() { setChecksumAndBodyLength(logon); LOGGER.info("Send logon - {}", logon); + logonResponded.set(false); channel.send(Unpooled.wrappedBuffer(logon.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } From b39bfeeb1622094fa882bd27d6d5d5470ad7041a Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 15:00:08 +0400 Subject: [PATCH 14/40] sequence reset instead of heartbeat for not captured messages in outgoing log --- .../java/com/exactpro/th2/FixHandler.java | 48 ++++++++----------- .../com/exactpro/th2/FixHandlerSettings.java | 10 ++-- .../exactpro/th2/util/FileDeserializer.java | 31 ++++++++++++ .../th2/util/LocalTimeDeserializer.java | 2 - src/main/java/com/exactpro/th2/util/Util.java | 11 +++-- 5 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 src/main/java/com/exactpro/th2/util/FileDeserializer.java diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 179aa64..73ad03f 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -81,6 +81,7 @@ import static com.exactpro.th2.constants.Constants.ENCRYPT_METHOD; import static com.exactpro.th2.constants.Constants.END_SEQ_NO; import static com.exactpro.th2.constants.Constants.END_SEQ_NO_TAG; +import static com.exactpro.th2.constants.Constants.GAP_FILL_FLAG; import static com.exactpro.th2.constants.Constants.GAP_FILL_FLAG_TAG; import static com.exactpro.th2.constants.Constants.HEART_BT_INT; import static com.exactpro.th2.constants.Constants.IS_POSS_DUP; @@ -118,7 +119,6 @@ import static com.exactpro.th2.constants.Constants.USERNAME; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.indexOf; import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.isEmpty; -import static com.exactpro.th2.netty.bytebuf.util.ByteBufUtil.set; import static com.exactpro.th2.util.MessageUtil.findByte; import static java.nio.charset.StandardCharsets.US_ASCII; import static java.util.Objects.requireNonNull; @@ -142,7 +142,6 @@ public class FixHandler implements AutoCloseable, IHandler { private final AtomicInteger testReqID = new AtomicInteger(0); private final AtomicBoolean enabled = new AtomicBoolean(false); private final AtomicBoolean connStarted = new AtomicBoolean(false); - private final AtomicBoolean logonResponded = new AtomicBoolean(true); private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); private final IHandlerContext context; private final InetSocketAddress address; @@ -157,7 +156,7 @@ public class FixHandler implements AutoCloseable, IHandler { public FixHandler(IHandlerContext context) { this.context = context; this.settings = (FixHandlerSettings) context.getSettings(); - if(settings.getStateFilePath() == null || !new File(settings.getStateFilePath()).exists()) { + if(settings.getStateFilePath() == null || !settings.getStateFilePath().exists()) { msgSeqNum = new AtomicInteger(0); serverMsgSeqNum = new AtomicInteger(0); } else { @@ -169,27 +168,25 @@ public FixHandler(IHandlerContext context) { Objects.requireNonNull(settings.getSessionEndTime(), "Session end is required when session start is presented"); LocalTime resetTime = settings.getSessionStartTime(); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); - ZonedDateTime scheduleTime; - if(now.with(resetTime).isAfter(now)) { + ZonedDateTime scheduleTime = now.with(resetTime); + if(scheduleTime.isBefore(now)) { scheduleTime = now.with(resetTime); - } else { - scheduleTime = now.plusDays(1).with(resetTime); } - long time = now.until(scheduleTime, ChronoUnit.HOURS); - executorService.scheduleAtFixedRate(this::reset, time, 24, TimeUnit.HOURS); + long time = now.until(scheduleTime, ChronoUnit.MINUTES); + executorService.scheduleAtFixedRate(this::reset, time, 24 * 60, TimeUnit.MINUTES); } if(settings.getSessionEndTime() != null) { LocalTime resetTime = settings.getSessionEndTime(); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); - ZonedDateTime scheduleTime; - if(now.with(resetTime).isAfter(now)) { - scheduleTime = now.with(resetTime); - } else { + ZonedDateTime scheduleTime = now.with(resetTime); + + if(scheduleTime.isBefore(now)) { scheduleTime = now.plusDays(1).with(resetTime); } - long time = now.until(scheduleTime, ChronoUnit.HOURS); - executorService.scheduleAtFixedRate(this::close, time, 24, TimeUnit.HOURS); + + long time = now.until(scheduleTime, ChronoUnit.MINUTES); + executorService.scheduleAtFixedRate(this::close, time, 24 * 60, TimeUnit.MINUTES); } String host = settings.getHost(); if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); @@ -293,11 +290,6 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu isDup = possDup.getValue().equals(IS_POSS_DUP); } - if(!logonResponded.get()) { - msgSeqNum.incrementAndGet(); - logonResponded.set(true); - } - int receivedMsgSeqNum = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); if(receivedMsgSeqNum < serverMsgSeqNum.get() && !isDup) { @@ -325,6 +317,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if (LOGGER.isInfoEnabled()) LOGGER.info("Logon received - {}", message.toString(US_ASCII)); boolean connectionSuccessful = checkLogon(message); if (connectionSuccessful) { + msgSeqNum.incrementAndGet(); if(settings.useNextExpectedSeqNum()) { FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); if(nextExpectedSeqField == null) { @@ -370,8 +363,8 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if (testRequestTimer != null) { testRequestTimer.cancel(false); } - msgSeqNum.getAndSet(Integer.parseInt(value) - 1); - serverMsgSeqNum.getAndSet(Integer.parseInt(msgSeqNumValue.getValue())); + msgSeqNum.set(Integer.parseInt(value) - 1); + serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue())); } } enabled.set(false); @@ -464,10 +457,12 @@ private void recovery(int beginSeqNo, int endSeqNo) { for (int i = beginSeqNo; i <= endSeqNo; i++) { ByteBuf storedMsg = outgoingMessages.get(i); if (storedMsg == null) { - StringBuilder heartbeat = new StringBuilder(); - setHeader(heartbeat, MSG_TYPE_HEARTBEAT, i); - setChecksumAndBodyLength(heartbeat); - channel.send(Unpooled.wrappedBuffer(heartbeat.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, SendMode.MANGLE); + StringBuilder sequenceReset = new StringBuilder(); + setHeader(sequenceReset, MSG_TYPE_SEQUENCE_RESET, i); + sequenceReset.append(GAP_FILL_FLAG).append("Y"); + sequenceReset.append(NEW_SEQ_NO).append(i); + setChecksumAndBodyLength(sequenceReset); + channel.send(Unpooled.wrappedBuffer(sequenceReset.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, SendMode.MANGLE); } else { if (LOGGER.isInfoEnabled()) LOGGER.info("Resending message: {}", storedMsg.toString(US_ASCII)); FixField sendingTime = findField(storedMsg, SENDING_TIME_TAG); @@ -710,7 +705,6 @@ public void sendLogon() { setChecksumAndBodyLength(logon); LOGGER.info("Send logon - {}", logon); - logonResponded.set(false); channel.send(Unpooled.wrappedBuffer(logon.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index f93be89..3f441de 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -19,10 +19,12 @@ import com.exactpro.th2.conn.dirty.fix.KeyFileType; import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel.Security; import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerSettings; +import com.exactpro.th2.util.FileDeserializer; import com.exactpro.th2.util.LocalTimeDeserializer; import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.io.File; import java.time.LocalTime; public class FixHandlerSettings implements IHandlerSettings { @@ -41,7 +43,9 @@ public class FixHandlerSettings implements IHandlerSettings { private String newPassword; private String passwordEncryptKeyFilePath; private KeyFileType passwordEncryptKeyFileType = KeyFileType.PEM_PUBLIC_KEY; - private String stateFilePath; + + @JsonDeserialize(using = FileDeserializer.class) + private File stateFilePath; /** * Value from Java Security Standard Algorithm Names */ @@ -213,11 +217,11 @@ public void setPasswordEncryptAlgorithm(String passwordEncryptAlgorithm) { this.passwordEncryptAlgorithm = passwordEncryptAlgorithm; } - public String getStateFilePath() { + public File getStateFilePath() { return stateFilePath; } - public void setStateFilePath(String stateFilePath) { + public void setStateFilePath(File stateFilePath) { this.stateFilePath = stateFilePath; } diff --git a/src/main/java/com/exactpro/th2/util/FileDeserializer.java b/src/main/java/com/exactpro/th2/util/FileDeserializer.java new file mode 100644 index 0000000..3814d7d --- /dev/null +++ b/src/main/java/com/exactpro/th2/util/FileDeserializer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.exactpro.th2.util; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.File; +import java.io.IOException; + +public class FileDeserializer extends StdDeserializer { + public FileDeserializer() {super(File.class);} + @Override + public File deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return new File(p.getValueAsString()); + } +} diff --git a/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java b/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java index 9e2376a..8cdb2d5 100644 --- a/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java +++ b/src/main/java/com/exactpro/th2/util/LocalTimeDeserializer.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.time.LocalTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; public class LocalTimeDeserializer extends StdDeserializer { diff --git a/src/main/java/com/exactpro/th2/util/Util.java b/src/main/java/com/exactpro/th2/util/Util.java index 0f6ea24..1dbed27 100644 --- a/src/main/java/com/exactpro/th2/util/Util.java +++ b/src/main/java/com/exactpro/th2/util/Util.java @@ -19,23 +19,24 @@ import java.io.BufferedReader; import java.io.BufferedWriter; +import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; public class Util { - public static SequenceHolder readSequences(String filePath) { - try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + public static SequenceHolder readSequences(File file) { + try (BufferedReader br = new BufferedReader(new FileReader(file))) { int firstLine = Integer.parseInt(br.readLine()); int secondLine = Integer.parseInt(br.readLine()); return new SequenceHolder(firstLine, secondLine); } catch (IOException e) { - throw new IllegalStateException("Error while reading sequence file " + filePath, e); + throw new IllegalStateException("Error while reading sequence file " + file, e); } } - public static void writeSequences(int msgSeqNum, int serverSeqNum, String filePath) throws IOException { - try (BufferedWriter bw = new BufferedWriter(new FileWriter(filePath))) { + public static void writeSequences(int msgSeqNum, int serverSeqNum, File file) throws IOException { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) { bw.write(String.valueOf(msgSeqNum)); bw.newLine(); bw.write(String.valueOf(serverSeqNum)); From 9e809d61df33c454be4df42f0a9ee2a0441e5001 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 16:58:07 +0400 Subject: [PATCH 15/40] temporarely add ability to change start sequence for client and server from settings. --- .../java/com/exactpro/th2/FixHandler.java | 12 ++++++++++-- .../com/exactpro/th2/FixHandlerSettings.java | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 73ad03f..9ddb934 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -157,13 +157,21 @@ public FixHandler(IHandlerContext context) { this.context = context; this.settings = (FixHandlerSettings) context.getSettings(); if(settings.getStateFilePath() == null || !settings.getStateFilePath().exists()) { - msgSeqNum = new AtomicInteger(0); - serverMsgSeqNum = new AtomicInteger(0); + msgSeqNum = new AtomicInteger(settings.getStartClientSeqNum()); + serverMsgSeqNum = new AtomicInteger(settings.getStartServerSeqNum()); } else { SequenceHolder sequences = Util.readSequences(settings.getStateFilePath()); msgSeqNum = new AtomicInteger(sequences.getClientSeq()); serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); } + if(settings.getStartClientSeqNum() != null) { + msgSeqNum.set(settings.getStartClientSeqNum()); + } + + if(settings.getStartServerSeqNum() != null) { + serverMsgSeqNum.set(settings.getStartServerSeqNum()); + } + if(settings.getSessionStartTime() != null) { Objects.requireNonNull(settings.getSessionEndTime(), "Session end is required when session start is presented"); LocalTime resetTime = settings.getSessionStartTime(); diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index 3f441de..d4426bb 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -44,6 +44,9 @@ public class FixHandlerSettings implements IHandlerSettings { private String passwordEncryptKeyFilePath; private KeyFileType passwordEncryptKeyFileType = KeyFileType.PEM_PUBLIC_KEY; + private Integer startClientSeqNum; + private Integer startServerSeqNum; + @JsonDeserialize(using = FileDeserializer.class) private File stateFilePath; /** @@ -141,6 +144,22 @@ public void setSenderSubID(String senderSubID) { this.senderSubID = senderSubID; } + public Integer getStartClientSeqNum() { + return startClientSeqNum; + } + + public void setStartClientSeqNum(Integer startClientSeqNum) { + this.startClientSeqNum = startClientSeqNum; + } + + public Integer getStartServerSeqNum() { + return startServerSeqNum; + } + + public void setStartServerSeqNum(Integer startServerSeqNum) { + this.startServerSeqNum = startServerSeqNum; + } + public String getEncryptMethod() { return encryptMethod; } From 219a026998cfcab6a30a0ba2ec7f875e492a9cfb Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 17:12:19 +0400 Subject: [PATCH 16/40] use seconds precision for session time tasks --- .../java/com/exactpro/th2/FixHandler.java | 12 +++---- .../com/exactpro/th2/FixHandlerSettings.java | 3 -- .../exactpro/th2/util/FileDeserializer.java | 31 ------------------- 3 files changed, 6 insertions(+), 40 deletions(-) delete mode 100644 src/main/java/com/exactpro/th2/util/FileDeserializer.java diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 8acbdef..c352f9f 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -157,8 +157,8 @@ public FixHandler(IHandlerContext context) { this.context = context; this.settings = (FixHandlerSettings) context.getSettings(); if(settings.getStateFilePath() == null || !settings.getStateFilePath().exists()) { - msgSeqNum = new AtomicInteger(settings.getStartClientSeqNum()); - serverMsgSeqNum = new AtomicInteger(settings.getStartServerSeqNum()); + msgSeqNum = new AtomicInteger(0); + serverMsgSeqNum = new AtomicInteger(0); } else { SequenceHolder sequences = Util.readSequences(settings.getStateFilePath()); msgSeqNum = new AtomicInteger(sequences.getClientSeq()); @@ -180,8 +180,8 @@ public FixHandler(IHandlerContext context) { if(scheduleTime.isBefore(now)) { scheduleTime = now.with(resetTime); } - long time = now.until(scheduleTime, ChronoUnit.MINUTES); - executorService.scheduleAtFixedRate(this::reset, time, 24 * 60, TimeUnit.MINUTES); + long time = now.until(scheduleTime, ChronoUnit.SECONDS); + executorService.scheduleAtFixedRate(this::reset, time, 24 * 60 * 60, TimeUnit.MINUTES); } if(settings.getSessionEndTime() != null) { @@ -193,8 +193,8 @@ public FixHandler(IHandlerContext context) { scheduleTime = now.plusDays(1).with(resetTime); } - long time = now.until(scheduleTime, ChronoUnit.MINUTES); - executorService.scheduleAtFixedRate(this::close, time, 24 * 60, TimeUnit.MINUTES); + long time = now.until(scheduleTime, ChronoUnit.SECONDS); + executorService.scheduleAtFixedRate(this::close, time, 24 * 60 * 60, TimeUnit.MINUTES); } String host = settings.getHost(); if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index d4426bb..809ef1f 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -19,9 +19,7 @@ import com.exactpro.th2.conn.dirty.fix.KeyFileType; import com.exactpro.th2.conn.dirty.tcp.core.api.IChannel.Security; import com.exactpro.th2.conn.dirty.tcp.core.api.IHandlerSettings; -import com.exactpro.th2.util.FileDeserializer; import com.exactpro.th2.util.LocalTimeDeserializer; -import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.io.File; @@ -47,7 +45,6 @@ public class FixHandlerSettings implements IHandlerSettings { private Integer startClientSeqNum; private Integer startServerSeqNum; - @JsonDeserialize(using = FileDeserializer.class) private File stateFilePath; /** * Value from Java Security Standard Algorithm Names diff --git a/src/main/java/com/exactpro/th2/util/FileDeserializer.java b/src/main/java/com/exactpro/th2/util/FileDeserializer.java deleted file mode 100644 index 3814d7d..0000000 --- a/src/main/java/com/exactpro/th2/util/FileDeserializer.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2023 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.exactpro.th2.util; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; - -import java.io.File; -import java.io.IOException; - -public class FileDeserializer extends StdDeserializer { - public FileDeserializer() {super(File.class);} - @Override - public File deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return new File(p.getValueAsString()); - } -} From 67bb9147f7adea18e608dffda133d04100b866b7 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 18:50:07 +0400 Subject: [PATCH 17/40] create file if not exists --- src/main/java/com/exactpro/th2/FixHandler.java | 4 +++- src/main/java/com/exactpro/th2/util/Util.java | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index c352f9f..a803c28 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -301,7 +301,9 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu int receivedMsgSeqNum = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); if(receivedMsgSeqNum < serverMsgSeqNum.get() && !isDup) { - sendLogout(); + if(enabled.get()) { + sendLogout(); + } reconnectRequestTimer = executorService.schedule(this::sendLogon, settings.getReconnectDelay(), TimeUnit.SECONDS); metadata.put(REJECT_REASON, "SeqNum is less than expected."); if (LOGGER.isErrorEnabled()) LOGGER.error("Invalid message. SeqNum is less than expected {}: {}", serverMsgSeqNum.get(), message.toString(US_ASCII)); diff --git a/src/main/java/com/exactpro/th2/util/Util.java b/src/main/java/com/exactpro/th2/util/Util.java index 1dbed27..d94b6c2 100644 --- a/src/main/java/com/exactpro/th2/util/Util.java +++ b/src/main/java/com/exactpro/th2/util/Util.java @@ -36,6 +36,10 @@ public static SequenceHolder readSequences(File file) { } public static void writeSequences(int msgSeqNum, int serverSeqNum, File file) throws IOException { + if(!file.exists()) { + file.mkdirs(); + file.createNewFile(); + } try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) { bw.write(String.valueOf(msgSeqNum)); bw.newLine(); From a17bab21c6cc2bdd4d794c73c9049ee7b7704aed Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:00:53 +0400 Subject: [PATCH 18/40] correct recovery logic --- src/main/java/com/exactpro/th2/FixHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index a803c28..99328a7 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -470,7 +470,7 @@ private void recovery(int beginSeqNo, int endSeqNo) { StringBuilder sequenceReset = new StringBuilder(); setHeader(sequenceReset, MSG_TYPE_SEQUENCE_RESET, i); sequenceReset.append(GAP_FILL_FLAG).append("Y"); - sequenceReset.append(NEW_SEQ_NO).append(i); + sequenceReset.append(NEW_SEQ_NO).append(i + 1); setChecksumAndBodyLength(sequenceReset); channel.send(Unpooled.wrappedBuffer(sequenceReset.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, SendMode.MANGLE); } else { From 414100a1755c60941b86fba8936eeee5fd5aab33 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:08:20 +0400 Subject: [PATCH 19/40] add loaded sequences logs --- src/main/java/com/exactpro/th2/FixHandler.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 99328a7..60da4ea 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -172,6 +172,8 @@ public FixHandler(IHandlerContext context) { serverMsgSeqNum.set(settings.getStartServerSeqNum()); } + LOGGER.info("Initial sequences are: client - {}, server - {}", msgSeqNum.get(), serverMsgSeqNum.get()); + if(settings.getSessionStartTime() != null) { Objects.requireNonNull(settings.getSessionEndTime(), "Session end is required when session start is presented"); LocalTime resetTime = settings.getSessionStartTime(); From ee971c8ae4e4be9d42c10b607d9240c69b3452d6 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:10:16 +0400 Subject: [PATCH 20/40] also log on writing sequence to state file --- src/main/java/com/exactpro/th2/FixHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 60da4ea..284ed51 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -730,6 +730,7 @@ private void sendLogout() { } if(settings.getStateFilePath() != null) { try { + LOGGER.info("Saving sequences: client - {}, server - {}", msgSeqNum.get(), serverMsgSeqNum.get() + 1); Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.get() + 1, settings.getStateFilePath()); } catch (IOException e) { if (LOGGER.isErrorEnabled()) { From 48a7436305054dc5e5dee9f4813ff5520586d2c8 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:30:29 +0400 Subject: [PATCH 21/40] Process sequence reset in gap fill mode correctly --- src/main/java/com/exactpro/th2/FixHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 284ed51..1a07916 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -376,7 +376,6 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu testRequestTimer.cancel(false); } msgSeqNum.set(Integer.parseInt(value) - 1); - serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue())); } } enabled.set(false); @@ -407,6 +406,8 @@ private void resetSequence(ByteBuf message) { if (seqNumValue != null && (gapFillFlagValue == null || requireNonNull(gapFillFlagValue.getValue()).equals("N"))) { serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue()))); + } else if(seqNumValue != null) { + serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue()))); } else if (LOGGER.isTraceEnabled()) { LOGGER.trace("Failed to reset servers MsgSeqNum. No such tag in message: {}", message.toString(US_ASCII)); } From a1d7f80dc0ba5420901bad2d21b596578f5fb419 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:39:29 +0400 Subject: [PATCH 22/40] Add error to log --- src/main/java/com/exactpro/th2/FixHandler.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 1a07916..ec5b69c 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -303,9 +303,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu int receivedMsgSeqNum = Integer.parseInt(requireNonNull(msgSeqNumValue.getValue())); if(receivedMsgSeqNum < serverMsgSeqNum.get() && !isDup) { - if(enabled.get()) { - sendLogout(); - } + sendLogout(); reconnectRequestTimer = executorService.schedule(this::sendLogon, settings.getReconnectDelay(), TimeUnit.SECONDS); metadata.put(REJECT_REASON, "SeqNum is less than expected."); if (LOGGER.isErrorEnabled()) LOGGER.error("Invalid message. SeqNum is less than expected {}: {}", serverMsgSeqNum.get(), message.toString(US_ASCII)); @@ -735,7 +733,7 @@ private void sendLogout() { Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.get() + 1, settings.getStateFilePath()); } catch (IOException e) { if (LOGGER.isErrorEnabled()) { - LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get()); + LOGGER.error("Sequence number were not saved: clientSeq - {}, serverSeq - {}", msgSeqNum.get(), serverMsgSeqNum.get(), e); } } } From b5356563a3707239cc04f1f695e5d359fc8ea418 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:54:03 +0400 Subject: [PATCH 23/40] create file if it not exist --- src/main/java/com/exactpro/th2/util/Util.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/util/Util.java b/src/main/java/com/exactpro/th2/util/Util.java index d94b6c2..fd4ca47 100644 --- a/src/main/java/com/exactpro/th2/util/Util.java +++ b/src/main/java/com/exactpro/th2/util/Util.java @@ -37,7 +37,7 @@ public static SequenceHolder readSequences(File file) { public static void writeSequences(int msgSeqNum, int serverSeqNum, File file) throws IOException { if(!file.exists()) { - file.mkdirs(); + file.getParentFile().mkdirs(); file.createNewFile(); } try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) { From 37a4f83070b8e26a384075bf14bfe3dc3e991494 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 19:58:22 +0400 Subject: [PATCH 24/40] update how state file is created --- src/main/java/com/exactpro/th2/util/Util.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/exactpro/th2/util/Util.java b/src/main/java/com/exactpro/th2/util/Util.java index fd4ca47..59458fb 100644 --- a/src/main/java/com/exactpro/th2/util/Util.java +++ b/src/main/java/com/exactpro/th2/util/Util.java @@ -36,10 +36,10 @@ public static SequenceHolder readSequences(File file) { } public static void writeSequences(int msgSeqNum, int serverSeqNum, File file) throws IOException { - if(!file.exists()) { + if(!file.getParentFile().exists()) { file.getParentFile().mkdirs(); - file.createNewFile(); } + file.createNewFile(); try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) { bw.write(String.valueOf(msgSeqNum)); bw.newLine(); From d243c3cb85cebfaf3dc2f46d1484ed902a760c1e Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 20:08:51 +0400 Subject: [PATCH 25/40] redefine server seq on logout failure --- src/main/java/com/exactpro/th2/FixHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index ec5b69c..f77d9d6 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -374,6 +374,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu testRequestTimer.cancel(false); } msgSeqNum.set(Integer.parseInt(value) - 1); + serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue())); } } enabled.set(false); From ac031cd2a42a3a48b07c4e863a466da28663aec8 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 20:34:52 +0400 Subject: [PATCH 26/40] write sequence right in constructor for debug purposes. --- src/main/java/com/exactpro/th2/FixHandler.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index f77d9d6..6650c51 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -163,6 +163,12 @@ public FixHandler(IHandlerContext context) { SequenceHolder sequences = Util.readSequences(settings.getStateFilePath()); msgSeqNum = new AtomicInteger(sequences.getClientSeq()); serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); + // TODO: delete + try { + Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.get(), settings.getStateFilePath()); + } catch (IOException e) { + LOGGER.error("Error while writing to file", e); + } } if(settings.getStartClientSeqNum() != null) { msgSeqNum.set(settings.getStartClientSeqNum()); From 109f61c5526cb5b595a2e81792c65751635f3ea3 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 21:32:00 +0400 Subject: [PATCH 27/40] remove temporary write --- src/main/java/com/exactpro/th2/FixHandler.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 6650c51..46c937e 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -163,13 +163,8 @@ public FixHandler(IHandlerContext context) { SequenceHolder sequences = Util.readSequences(settings.getStateFilePath()); msgSeqNum = new AtomicInteger(sequences.getClientSeq()); serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); - // TODO: delete - try { - Util.writeSequences(msgSeqNum.get(), serverMsgSeqNum.get(), settings.getStateFilePath()); - } catch (IOException e) { - LOGGER.error("Error while writing to file", e); - } } + if(settings.getStartClientSeqNum() != null) { msgSeqNum.set(settings.getStartClientSeqNum()); } @@ -727,11 +722,10 @@ public void sendLogon() { } private void sendLogout() { - StringBuilder logout = new StringBuilder(); - setHeader(logout, MSG_TYPE_LOGOUT, msgSeqNum.incrementAndGet()); - setChecksumAndBodyLength(logout); - if (enabled.get()) { + StringBuilder logout = new StringBuilder(); + setHeader(logout, MSG_TYPE_LOGOUT, msgSeqNum.incrementAndGet()); + setChecksumAndBodyLength(logout); channel.send(Unpooled.wrappedBuffer(logout.toString().getBytes(StandardCharsets.UTF_8)), Collections.emptyMap(), null, IChannel.SendMode.MANGLE); } if(settings.getStateFilePath() != null) { @@ -769,9 +763,7 @@ public void onClose(@NotNull IChannel channel) { @Override public void close() { - if(enabled.get()) { - sendLogout(); - } + sendLogout(); } private void setHeader(StringBuilder stringBuilder, String msgType, Integer seqNum) { From e0e1c9e02e65dd203b21cf83c10fd6d770a53492 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 21:47:42 +0400 Subject: [PATCH 28/40] update server seq num correctly --- src/main/java/com/exactpro/th2/FixHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 46c937e..4e92d76 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -375,7 +375,7 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu testRequestTimer.cancel(false); } msgSeqNum.set(Integer.parseInt(value) - 1); - serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue())); + serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue()) - 1); } } enabled.set(false); From 6326ed6ca0f463959e0e6f153b7d236ceb96bd78 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 20 Jan 2023 22:33:00 +0400 Subject: [PATCH 29/40] save corect server seq num --- src/main/java/com/exactpro/th2/FixHandler.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 4e92d76..13b7038 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -181,7 +181,7 @@ public FixHandler(IHandlerContext context) { ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime scheduleTime = now.with(resetTime); if(scheduleTime.isBefore(now)) { - scheduleTime = now.with(resetTime); + scheduleTime = now.plusDays(1).with(resetTime); } long time = now.until(scheduleTime, ChronoUnit.SECONDS); executorService.scheduleAtFixedRate(this::reset, time, 24 * 60 * 60, TimeUnit.MINUTES); @@ -591,7 +591,7 @@ public void onOutgoingUpdateTag(@NotNull ByteBuf message, @NotNull Map Date: Mon, 23 Jan 2023 14:24:05 +0400 Subject: [PATCH 30/40] Enforce 'session time start must be after session time end' --- src/main/java/com/exactpro/th2/FixHandler.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 13b7038..9489580 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -177,6 +177,9 @@ public FixHandler(IHandlerContext context) { if(settings.getSessionStartTime() != null) { Objects.requireNonNull(settings.getSessionEndTime(), "Session end is required when session start is presented"); + if(settings.getSessionStartTime().isBefore(settings.getSessionEndTime())) { + throw new IllegalStateException("Session end time must be before session start time in a timeline."); + } LocalTime resetTime = settings.getSessionStartTime(); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime scheduleTime = now.with(resetTime); @@ -199,6 +202,7 @@ public FixHandler(IHandlerContext context) { long time = now.until(scheduleTime, ChronoUnit.SECONDS); executorService.scheduleAtFixedRate(this::close, time, 24 * 60 * 60, TimeUnit.MINUTES); } + String host = settings.getHost(); if (host == null || host.isBlank()) throw new IllegalArgumentException("host cannot be blank"); int port = settings.getPort(); From 404be283b0842af6e0d578705813c0263d137c72 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Mon, 23 Jan 2023 14:31:40 +0400 Subject: [PATCH 31/40] remove temporary things and add new settings description to README --- README.md | 14 ++++++++++- gradle.properties | 2 +- .../java/com/exactpro/th2/FixHandler.java | 8 ------ .../com/exactpro/th2/FixHandlerSettings.java | 25 +++---------------- 4 files changed, 17 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5710726..de0f527 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# th2-conn-dirty-fix (0.0.2) +# th2-conn-dirty-fix (0.0.4) This microservice allows sending and receiving messages via FIX protocol @@ -41,6 +41,11 @@ This microservice allows sending and receiving messages via FIX protocol + *disconnectRequestDelay* - the interval for the shutdown request + *resetSeqNumFlag* - resetting sequence number in initial Logon message (when conn started) + *resetOnLogon* - resetting the sequence number in Logon in other cases (e.g. disconnect) ++ *stateFilePath* - path to file where sequences will be saved to use with next login attempts. It is useful when acceptor is not support sequence reset. (`nullable`) ++ *startSessionTime* - UTC time when session starts. (`nullable`) ++ *endSessionTime* - UTC time when session ends. required if startSessionTime is filled. ++ *useNextExpectedSeqNum* - session management based on next expected sequence number. (`false` by default) ++ *saveAdminMessages* - defines if admin messages will be saved to internal outgoing buffer. (`false` by default) ### Security settings @@ -301,6 +306,13 @@ spec: # Changelog +## 0.0.4 +* Session management based on NextExpectedSeqNum field. +* Recovery handling + * outgoing messages are now saved + * if message wasn't saved sequence reset message with gap fill mode flag is sent. +* Session start and Session end configuration to handle sequence reset by exchange schedule. + ## 0.0.3 * Added new password option into settings diff --git a/gradle.properties b/gradle.properties index 2259ea7..5fbfe13 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ -release_version=0.0.3 +release_version=0.0.4 jackson_version=2.11.2 \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 9489580..34ff693 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -165,14 +165,6 @@ public FixHandler(IHandlerContext context) { serverMsgSeqNum = new AtomicInteger(sequences.getServerSeq()); } - if(settings.getStartClientSeqNum() != null) { - msgSeqNum.set(settings.getStartClientSeqNum()); - } - - if(settings.getStartServerSeqNum() != null) { - serverMsgSeqNum.set(settings.getStartServerSeqNum()); - } - LOGGER.info("Initial sequences are: client - {}, server - {}", msgSeqNum.get(), serverMsgSeqNum.get()); if(settings.getSessionStartTime() != null) { diff --git a/src/main/java/com/exactpro/th2/FixHandlerSettings.java b/src/main/java/com/exactpro/th2/FixHandlerSettings.java index 809ef1f..673065f 100644 --- a/src/main/java/com/exactpro/th2/FixHandlerSettings.java +++ b/src/main/java/com/exactpro/th2/FixHandlerSettings.java @@ -42,9 +42,6 @@ public class FixHandlerSettings implements IHandlerSettings { private String passwordEncryptKeyFilePath; private KeyFileType passwordEncryptKeyFileType = KeyFileType.PEM_PUBLIC_KEY; - private Integer startClientSeqNum; - private Integer startServerSeqNum; - private File stateFilePath; /** * Value from Java Security Standard Algorithm Names @@ -57,7 +54,7 @@ public class FixHandlerSettings implements IHandlerSettings { private Boolean resetSeqNumFlag = false; private Boolean resetOnLogon = false; private Boolean useNextExpectedSeqNum = false; - private Boolean isSaveAdminMessages = false; + private Boolean saveAdminMessages = false; @JsonDeserialize(using = LocalTimeDeserializer.class) private LocalTime sessionStartTime; @@ -141,22 +138,6 @@ public void setSenderSubID(String senderSubID) { this.senderSubID = senderSubID; } - public Integer getStartClientSeqNum() { - return startClientSeqNum; - } - - public void setStartClientSeqNum(Integer startClientSeqNum) { - this.startClientSeqNum = startClientSeqNum; - } - - public Integer getStartServerSeqNum() { - return startServerSeqNum; - } - - public void setStartServerSeqNum(Integer startServerSeqNum) { - this.startServerSeqNum = startServerSeqNum; - } - public String getEncryptMethod() { return encryptMethod; } @@ -250,11 +231,11 @@ public void setUseNextExpectedSeqNum(Boolean useNextExpectedSeqNum) { } public Boolean isSaveAdminMessages() { - return isSaveAdminMessages; + return saveAdminMessages; } public void setSaveAdminMessages(Boolean saveAdminMessages) { - isSaveAdminMessages = saveAdminMessages; + this.saveAdminMessages = saveAdminMessages; } public LocalTime getSessionStartTime() { From 9aac2ebc5828df27c4e561d2a4d8dfb8a31d203d Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Tue, 24 Jan 2023 00:53:52 +0400 Subject: [PATCH 32/40] Prohibit messages send when session is not active --- README.md | 6 ++-- .../java/com/exactpro/th2/FixHandler.java | 34 ++++++++++++------- .../com/exactpro/th2/constants/Constants.java | 1 - 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index de0f527..b621b90 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,9 @@ This microservice allows sending and receiving messages via FIX protocol + *disconnectRequestDelay* - the interval for the shutdown request + *resetSeqNumFlag* - resetting sequence number in initial Logon message (when conn started) + *resetOnLogon* - resetting the sequence number in Logon in other cases (e.g. disconnect) -+ *stateFilePath* - path to file where sequences will be saved to use with next login attempts. It is useful when acceptor is not support sequence reset. (`nullable`) -+ *startSessionTime* - UTC time when session starts. (`nullable`) -+ *endSessionTime* - UTC time when session ends. required if startSessionTime is filled. ++ *stateFilePath* - path to file where sequences will be saved to use with next login attempts. It is useful when acceptor does not support sequence reset. (`nullable`) ++ *sessionStartTime* - UTC time when session starts. (`nullable`) ++ *sessionEndTime* - UTC time when session ends. required if startSessionTime is filled. + *useNextExpectedSeqNum* - session management based on next expected sequence number. (`false` by default) + *saveAdminMessages* - defines if admin messages will be saved to internal outgoing buffer. (`false` by default) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 34ff693..2f765e7 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -33,20 +33,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; -import java.time.Instant; import java.time.LocalDateTime; import java.time.LocalTime; -import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -130,6 +126,8 @@ public class FixHandler implements AutoCloseable, IHandler { private static final Logger LOGGER = LoggerFactory.getLogger(FixHandler.class); + + private static final int DAY_SECONDS = 24 * 60 * 60; private static final String SOH = "\001"; private static final byte BYTE_SOH = 1; private static final String STRING_MSG_TYPE = "MsgType"; @@ -140,6 +138,7 @@ public class FixHandler implements AutoCloseable, IHandler { private final AtomicInteger msgSeqNum; private final AtomicInteger serverMsgSeqNum; private final AtomicInteger testReqID = new AtomicInteger(0); + private final AtomicBoolean sessionActive = new AtomicBoolean(true); private final AtomicBoolean enabled = new AtomicBoolean(false); private final AtomicBoolean connStarted = new AtomicBoolean(false); private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); @@ -169,17 +168,15 @@ public FixHandler(IHandlerContext context) { if(settings.getSessionStartTime() != null) { Objects.requireNonNull(settings.getSessionEndTime(), "Session end is required when session start is presented"); - if(settings.getSessionStartTime().isBefore(settings.getSessionEndTime())) { - throw new IllegalStateException("Session end time must be before session start time in a timeline."); - } LocalTime resetTime = settings.getSessionStartTime(); ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime scheduleTime = now.with(resetTime); + if(scheduleTime.isBefore(now)) { scheduleTime = now.plusDays(1).with(resetTime); } long time = now.until(scheduleTime, ChronoUnit.SECONDS); - executorService.scheduleAtFixedRate(this::reset, time, 24 * 60 * 60, TimeUnit.MINUTES); + executorService.scheduleAtFixedRate(this::reset, time, DAY_SECONDS, TimeUnit.SECONDS); } if(settings.getSessionEndTime() != null) { @@ -189,10 +186,15 @@ public FixHandler(IHandlerContext context) { if(scheduleTime.isBefore(now)) { scheduleTime = now.plusDays(1).with(resetTime); + } else if(scheduleTime.isBefore(now.with(settings.getSessionStartTime()))) { + sessionActive.set(false); } long time = now.until(scheduleTime, ChronoUnit.SECONDS); - executorService.scheduleAtFixedRate(this::close, time, 24 * 60 * 60, TimeUnit.MINUTES); + executorService.scheduleAtFixedRate(() -> { + this.close(); + sessionActive.set(false); + }, time, DAY_SECONDS, TimeUnit.SECONDS); } String host = settings.getHost(); @@ -218,6 +220,9 @@ public void onStart() { @NotNull @Override public CompletableFuture send(@NotNull RawMessage rawMessage) { + if (!sessionActive.get()) { + throw new IllegalStateException("Session is not active. It is not possible to send messages."); + } if (!channel.isOpen()) { try { channel.open().get(); @@ -400,11 +405,9 @@ private void resetSequence(ByteBuf message) { FixField gapFillFlagValue = findField(message, GAP_FILL_FLAG_TAG); FixField seqNumValue = findField(message, NEW_SEQ_NO_TAG); - if (seqNumValue != null && (gapFillFlagValue == null || requireNonNull(gapFillFlagValue.getValue()).equals("N"))) { + if(seqNumValue != null) { serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue()))); - } else if(seqNumValue != null) { - serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue()))); - } else if (LOGGER.isTraceEnabled()) { + } else { LOGGER.trace("Failed to reset servers MsgSeqNum. No such tag in message: {}", message.toString(US_ASCII)); } } @@ -412,6 +415,7 @@ private void resetSequence(ByteBuf message) { private void reset() { msgSeqNum.set(0); serverMsgSeqNum.set(0); + sessionActive.set(true); sendLogon(); } @@ -683,6 +687,10 @@ public void sendTestRequest() { //do private } public void sendLogon() { + if(!sessionActive.get()) { + LOGGER.info("Logon is not sent to server because session is not active."); + return; + } lastSendTime = System.currentTimeMillis(); StringBuilder logon = new StringBuilder(); Boolean reset; diff --git a/src/main/java/com/exactpro/th2/constants/Constants.java b/src/main/java/com/exactpro/th2/constants/Constants.java index 588bc70..3af863c 100644 --- a/src/main/java/com/exactpro/th2/constants/Constants.java +++ b/src/main/java/com/exactpro/th2/constants/Constants.java @@ -97,5 +97,4 @@ public class Constants { ); public static final String IS_POSS_DUP = "Y"; - } From 26b4462059875c8c60ee715b3924bacc84e92f98 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Fri, 27 Jan 2023 14:49:20 +0400 Subject: [PATCH 33/40] correct sessionActive value when session is not active on start --- src/main/java/com/exactpro/th2/FixHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 2f765e7..83ac6a5 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -186,7 +186,7 @@ public FixHandler(IHandlerContext context) { if(scheduleTime.isBefore(now)) { scheduleTime = now.plusDays(1).with(resetTime); - } else if(scheduleTime.isBefore(now.with(settings.getSessionStartTime()))) { + } else if(now.isBefore(now.with(settings.getSessionStartTime()))) { sessionActive.set(false); } From ad2b351a326dca59371bcd24e4e2f51a9c83efef Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Mon, 30 Jan 2023 13:20:56 +0400 Subject: [PATCH 34/40] update seqnums properly --- .../java/com/exactpro/th2/FixHandler.java | 24 ++++++++++++++----- .../com/exactpro/th2/constants/Constants.java | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 83ac6a5..abd59d1 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -107,6 +107,7 @@ import static com.exactpro.th2.constants.Constants.SENDING_TIME; import static com.exactpro.th2.constants.Constants.SENDING_TIME_TAG; import static com.exactpro.th2.constants.Constants.SESSION_STATUS_TAG; +import static com.exactpro.th2.constants.Constants.SUCCESSFUL_LOGOUT_CODE; import static com.exactpro.th2.constants.Constants.TARGET_COMP_ID; import static com.exactpro.th2.constants.Constants.TARGET_COMP_ID_TAG; import static com.exactpro.th2.constants.Constants.TEST_REQ_ID; @@ -364,18 +365,29 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu break; case MSG_TYPE_LOGOUT: //extract logout reason if (LOGGER.isInfoEnabled()) LOGGER.info("Logout received - {}", message.toString(US_ASCII)); - FixField text = findField(message, TEXT_TAG); - if (text != null) { - LOGGER.warn("Received Logout has text (58) tag: {}", text.getValue()); - String value = StringUtils.substringBetween(text.getValue(), "expecting ", " but received"); - if (value != null) { + FixField sessionStatus = findField(message, SESSION_STATUS_TAG); + + if(sessionStatus != null) { + int statusCode = Integer.parseInt(Objects.requireNonNull(sessionStatus.getValue())); + if(statusCode != SUCCESSFUL_LOGOUT_CODE) { if (heartbeatTimer != null) { heartbeatTimer.cancel(false); } if (testRequestTimer != null) { testRequestTimer.cancel(false); } - msgSeqNum.set(Integer.parseInt(value) - 1); + FixField text = findField(message, TEXT_TAG); + if (text != null) { + LOGGER.warn("Received Logout has text (58) tag: {}", text.getValue()); + String value = StringUtils.substringBetween(text.getValue(), "expecting ", " but received"); + if (value != null) { + msgSeqNum.set(Integer.parseInt(value) - 1); + } else { + msgSeqNum.set(msgSeqNum.get() - 1); + } + } else { + msgSeqNum.set(msgSeqNum.get() - 1); + } serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue()) - 1); } } diff --git a/src/main/java/com/exactpro/th2/constants/Constants.java b/src/main/java/com/exactpro/th2/constants/Constants.java index 3af863c..f953b8f 100644 --- a/src/main/java/com/exactpro/th2/constants/Constants.java +++ b/src/main/java/com/exactpro/th2/constants/Constants.java @@ -97,4 +97,5 @@ public class Constants { ); public static final String IS_POSS_DUP = "Y"; + public static final int SUCCESSFUL_LOGOUT_CODE = 4; } From 67b5009a6e0bed2cf5f3cc82554290d8d468da00 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Mon, 30 Jan 2023 13:31:10 +0400 Subject: [PATCH 35/40] minor fix --- src/main/java/com/exactpro/th2/FixHandler.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index abd59d1..a761829 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -370,12 +370,6 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if(sessionStatus != null) { int statusCode = Integer.parseInt(Objects.requireNonNull(sessionStatus.getValue())); if(statusCode != SUCCESSFUL_LOGOUT_CODE) { - if (heartbeatTimer != null) { - heartbeatTimer.cancel(false); - } - if (testRequestTimer != null) { - testRequestTimer.cancel(false); - } FixField text = findField(message, TEXT_TAG); if (text != null) { LOGGER.warn("Received Logout has text (58) tag: {}", text.getValue()); @@ -391,6 +385,12 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu serverMsgSeqNum.set(Integer.parseInt(msgSeqNumValue.getValue()) - 1); } } + if (heartbeatTimer != null) { + heartbeatTimer.cancel(false); + } + if (testRequestTimer != null) { + testRequestTimer.cancel(false); + } enabled.set(false); context.send(CommonUtil.toEvent("logout for sender - " + settings.getSenderCompID()));//make more useful break; From a6bf8b75df1456d0f53393a0ef63a3d4d78f9145 Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Tue, 31 Jan 2023 17:44:30 +0400 Subject: [PATCH 36/40] correct sequence reset handling --- src/main/java/com/exactpro/th2/FixHandler.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index a761829..33a4dbb 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -414,11 +414,15 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu } private void resetSequence(ByteBuf message) { - FixField gapFillFlagValue = findField(message, GAP_FILL_FLAG_TAG); + FixField gapFillMode = findField(message, GAP_FILL_FLAG_TAG); FixField seqNumValue = findField(message, NEW_SEQ_NO_TAG); if(seqNumValue != null) { - serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue()))); + if(gapFillMode == null || gapFillMode.equals("N")) { + serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue()))); + } else { + serverMsgSeqNum.set(Integer.parseInt(requireNonNull(seqNumValue.getValue())) - 1); + } } else { LOGGER.trace("Failed to reset servers MsgSeqNum. No such tag in message: {}", message.toString(US_ASCII)); } @@ -699,7 +703,7 @@ public void sendTestRequest() { //do private } public void sendLogon() { - if(!sessionActive.get()) { + if(!sessionActive.get() || !channel.isOpen()) { LOGGER.info("Logon is not sent to server because session is not active."); return; } @@ -710,7 +714,7 @@ public void sendLogon() { else reset = settings.getResetOnLogon(); if (reset) msgSeqNum.getAndSet(0); - setHeader(logon, MSG_TYPE_LOGON, msgSeqNum.get() + 1); + setHeader(logon, MSG_TYPE_LOGON, msgSeqNum.incrementAndGet()); if (settings.useNextExpectedSeqNum()) logon.append(NEXT_EXPECTED_SEQ_NUM).append(serverMsgSeqNum.get() + 1); if (settings.getEncryptMethod() != null) logon.append(ENCRYPT_METHOD).append(settings.getEncryptMethod()); logon.append(HEART_BT_INT).append(settings.getHeartBtInt()); From 4077e266e4bc5b638e23ae144841563f3f08f31e Mon Sep 17 00:00:00 2001 From: Denis Plotnikov Date: Tue, 31 Jan 2023 18:00:38 +0400 Subject: [PATCH 37/40] fix tests --- src/main/java/com/exactpro/th2/FixHandler.java | 1 - src/test/java/com/exactpro/th2/FixHandlerTest.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/exactpro/th2/FixHandler.java b/src/main/java/com/exactpro/th2/FixHandler.java index 33a4dbb..6838801 100644 --- a/src/main/java/com/exactpro/th2/FixHandler.java +++ b/src/main/java/com/exactpro/th2/FixHandler.java @@ -330,7 +330,6 @@ public Map onIncoming(@NotNull IChannel channel, @NotNull ByteBu if (LOGGER.isInfoEnabled()) LOGGER.info("Logon received - {}", message.toString(US_ASCII)); boolean connectionSuccessful = checkLogon(message); if (connectionSuccessful) { - msgSeqNum.incrementAndGet(); if(settings.useNextExpectedSeqNum()) { FixField nextExpectedSeqField = findField(message, NEXT_EXPECTED_SEQ_NUMBER_TAG); if(nextExpectedSeqField == null) { diff --git a/src/test/java/com/exactpro/th2/FixHandlerTest.java b/src/test/java/com/exactpro/th2/FixHandlerTest.java index 255f51b..ece4a57 100644 --- a/src/test/java/com/exactpro/th2/FixHandlerTest.java +++ b/src/test/java/com/exactpro/th2/FixHandlerTest.java @@ -368,7 +368,7 @@ public CompletableFuture send(@NotNull ByteBuf byteBuf, @NotNull Map< @Override public boolean isOpen() { - return false; + return true; } @Override From f32340029030f728acbd620b95d53f0babd8443c Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Mon, 6 Feb 2023 12:30:55 +0400 Subject: [PATCH 38/40] [TH2-4347] excluded vulnerable log4j dependency --- build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d0efdbb..37b3ee7 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,10 @@ dependencies { implementation 'com.exactpro.th2:common:3.44.0' implementation 'com.exactpro.th2:netty-bytebuf-utils:0.0.1' - implementation 'com.exactpro.th2:conn-dirty-tcp-core:2.0.4' + implementation ('com.exactpro.th2:conn-dirty-tcp-core:2.0.5') { + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + because 'Projects should use only slf4j-api, without coupling to a sertain implementation' + } implementation 'org.slf4j:slf4j-api:1.7.33' implementation 'io.github.microutils:kotlin-logging:2.1.23' From 066be7246bebeba93076b70e6fd8f67d4e64e292 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Tue, 7 Feb 2023 14:31:06 +0400 Subject: [PATCH 39/40] slf4j-api update --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 37b3ee7..ff5d94f 100644 --- a/build.gradle +++ b/build.gradle @@ -53,7 +53,7 @@ dependencies { because 'Projects should use only slf4j-api, without coupling to a sertain implementation' } - implementation 'org.slf4j:slf4j-api:1.7.33' + implementation 'org.slf4j:slf4j-api' implementation 'io.github.microutils:kotlin-logging:2.1.23' implementation 'io.netty:netty-all:4.1.86.Final' From 513e7e3a9736210674f89ab07b1e4c974833c115 Mon Sep 17 00:00:00 2001 From: "fiodar.rekish" Date: Fri, 10 Feb 2023 15:43:18 +0400 Subject: [PATCH 40/40] readme fix --- README.md | 20 +++++++++++++++++--- build.gradle | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b621b90..9c7220c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ This microservice allows sending and receiving messages via FIX protocol + *sessions* - list of session settings + *maxBatchSize* - max size of outgoing message batch (`1000` by default) + *maxFlushTime* - max message batch flush time (`1000` by default) -+ *batchByGroup* - batch messages by group instead of session alias and direction (`true` by default) + *publishSentEvents* - enables/disables publish of "message sent" events (`true` by default) + *publishConnectEvents* - enables/disables publish of "connect/disconnect" events (`true` by default) @@ -226,7 +225,6 @@ spec: custom-config: maxBatchSize: 1000 maxFlushTime: 1000 - batchByGroup: false publishSentEvents: true publishConnectEvents: true sessions: @@ -284,12 +282,28 @@ spec: settings: storageOnDemand: false queueLength: 1000 - - name: outgoing + - name: incoming_messages connection-type: mq attributes: - publish - store - raw + filters: + - metadata: + - field-name: direction + expected-value: FIRST + operation: EQUAL + - name: outgoing_messages + connection-type: mq + attributes: + - publish + - store + - raw + filters: + - metadata: + - field-name: direction + expected-value: SECOND + operation: EQUAL extended-settings: externalBox: enabled: false diff --git a/build.gradle b/build.gradle index ff5d94f..b88c8a4 100644 --- a/build.gradle +++ b/build.gradle @@ -50,7 +50,7 @@ dependencies { implementation 'com.exactpro.th2:netty-bytebuf-utils:0.0.1' implementation ('com.exactpro.th2:conn-dirty-tcp-core:2.0.5') { exclude group: 'org.slf4j', module: 'slf4j-log4j12' - because 'Projects should use only slf4j-api, without coupling to a sertain implementation' + because 'Projects should use only slf4j-api, without coupling to a certain implementation' } implementation 'org.slf4j:slf4j-api'