+spring.datasource.username=postgres
+spring.datasource.password=postgres
+spring.datasource.hikari.maximum-pool-size=8
+spring.datasource.hikari.auto-commit=false
+```
+
+### Disable elasticsearch indexing
+```properties
+conductor.app.asyncIndexingEnabled=false
+conductor.indexing.enabled=false
+```
diff --git a/archive/build.gradle b/archive/build.gradle
new file mode 100644
index 0000000..2332c73
--- /dev/null
+++ b/archive/build.gradle
@@ -0,0 +1,51 @@
+dependencies {
+
+ implementation "com.netflix.conductor:conductor-common:${versions.conductorfork}"
+ implementation "com.netflix.conductor:conductor-core:${versions.conductorfork}"
+ implementation ("com.netflix.conductor:conductor-postgres-persistence:${versions.conductorfork}")
+
+ implementation 'org.springframework.boot:spring-boot-starter'
+ implementation 'org.springframework.boot:spring-boot-starter-data-jdbc'
+ implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
+ implementation 'org.springframework.boot:spring-boot-starter-security'
+
+ implementation "com.fasterxml.jackson.core:jackson-databind"
+ implementation "com.fasterxml.jackson.core:jackson-core"
+
+ //Metrics
+ implementation "io.micrometer:micrometer-registry-prometheus:1.7.5"
+ implementation "io.micrometer:micrometer-core:1.7.5"
+ implementation "com.netflix.spectator:spectator-reg-micrometer:${versions.revSpectator}"
+ implementation "com.netflix.spectator:spectator-reg-metrics3:${versions.revSpectator}"
+ implementation "com.netflix.spectator:spectator-api:${versions.revSpectator}"
+
+ implementation 'org.springframework.retry:spring-retry'
+
+ //Flyway for postgres configuration
+ implementation "org.flywaydb:flyway-core"
+
+ //Cache
+ implementation "com.google.guava:guava:${versions.revGuava}"
+
+ //Lucene
+ implementation "org.apache.lucene:lucene-core:${versions.revLucene}"
+ implementation "org.apache.lucene:lucene-analyzers-common:${versions.revLucene}"
+
+ //spring
+ testImplementation 'org.springframework.boot:spring-boot-starter-test'
+ testImplementation 'org.springframework.security:spring-security-test'
+
+
+ testImplementation 'org.hamcrest:hamcrest'
+ testImplementation "org.awaitility:awaitility:3.1.6"
+
+ //postgres test container
+ testImplementation "org.testcontainers:postgresql:${versions.revTestContainer}"
+
+ //Fake data generator
+ testImplementation "com.github.javafaker:javafaker:1.0.2"
+}
+
+test {
+ useJUnitPlatform()
+}
\ No newline at end of file
diff --git a/archive/src/main/java/io/orkes/conductor/dao/archive/ArchiveDAO.java b/archive/src/main/java/io/orkes/conductor/dao/archive/ArchiveDAO.java
new file mode 100644
index 0000000..58bc968
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/archive/ArchiveDAO.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.archive;
+
+import java.util.List;
+
+import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
+import com.netflix.conductor.model.WorkflowModel;
+
+public interface ArchiveDAO {
+
+ // Workflow Methods
+
+ void createOrUpdateWorkflow(WorkflowModel workflow);
+
+ boolean removeWorkflow(String workflowId);
+
+ WorkflowModel getWorkflow(String workflowId, boolean includeTasks);
+
+ List getWorkflowIdsByType(String workflowName, Long startTime, Long endTime);
+
+ List getWorkflowIdsByCorrelationId(
+ String workflowName, String correlationId, boolean includeClosed, boolean includeTasks);
+
+ ScrollableSearchResult searchWorkflows(
+ String query, String freeText, int start, int count);
+
+ List getTaskExecutionLogs(String taskId);
+
+ void addTaskExecutionLogs(List logs);
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/archive/ArchivedExecutionDAO.java b/archive/src/main/java/io/orkes/conductor/dao/archive/ArchivedExecutionDAO.java
new file mode 100644
index 0000000..d73409e
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/archive/ArchivedExecutionDAO.java
@@ -0,0 +1,345 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.archive;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.Collectors;
+
+import com.netflix.conductor.common.metadata.events.EventExecution;
+import com.netflix.conductor.common.run.Workflow;
+import com.netflix.conductor.dao.ExecutionDAO;
+import com.netflix.conductor.dao.QueueDAO;
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+
+import io.orkes.conductor.dao.indexer.IndexWorker;
+import io.orkes.conductor.id.TimeBasedUUIDGenerator;
+import io.orkes.conductor.metrics.MetricsCollector;
+
+import lombok.extern.slf4j.Slf4j;
+
+import static io.orkes.conductor.dao.indexer.IndexWorker.INDEXER_QUEUE;
+
+@Slf4j
+public class ArchivedExecutionDAO implements ExecutionDAO {
+
+ private static final int OFFSET_TIME_SEC = 0;
+
+ private final ExecutionDAO primaryDAO;
+
+ private final ArchiveDAO archiveDAO;
+
+ private final QueueDAO queueDAO;
+
+ private final MetricsCollector metricsCollector;
+
+ private final Clock clock;
+
+ public ArchivedExecutionDAO(
+ ExecutionDAO primaryDAO,
+ ArchiveDAO archiveDAO,
+ QueueDAO queueDAO,
+ MetricsCollector metricsCollector) {
+ this.primaryDAO = primaryDAO;
+ this.archiveDAO = archiveDAO;
+ this.queueDAO = queueDAO;
+ this.metricsCollector = metricsCollector;
+ this.clock = Clock.systemDefaultZone();
+ log.info(
+ "Initialized {} as Execution DAO with {} as primary DAO",
+ ArchivedExecutionDAO.class.getSimpleName(),
+ primaryDAO.getClass().getSimpleName());
+ }
+
+ ////////////////////////////////////////////////////////////////////////
+ // Delegate to Primary DAO //
+ ////////////////////////////////////////////////////////////////////////
+ @Override
+ public List getPendingTasksByWorkflow(String taskName, String workflowId) {
+ return primaryDAO.getPendingTasksByWorkflow(taskName, workflowId);
+ }
+
+ @Override
+ public List createTasks(List tasks) {
+ return metricsCollector
+ .getTimer("create_tasks_dao")
+ .record(() -> primaryDAO.createTasks(tasks));
+ }
+
+ @Override
+ public void updateTask(TaskModel task) {
+ metricsCollector
+ .getTimer("update_task_dao", "taskType", task.getTaskDefName())
+ .record(
+ () -> {
+ primaryDAO.updateTask(task);
+ if (task.getStatus().isTerminal()) {
+ metricsCollector.recordTaskComplete(task);
+ }
+ });
+ }
+
+ @Override
+ public boolean exceedsInProgressLimit(TaskModel task) {
+ return primaryDAO.exceedsInProgressLimit(task);
+ }
+
+ @Override
+ public boolean removeTask(String taskId) {
+ return primaryDAO.removeTask(taskId);
+ }
+
+ @Override
+ public List getPendingTasksForTaskType(String taskType) {
+ return primaryDAO.getPendingTasksForTaskType(taskType);
+ }
+
+ @Override
+ public void removeFromPendingWorkflow(String workflowType, String workflowId) {
+ primaryDAO.removeFromPendingWorkflow(workflowType, workflowId);
+ }
+
+ @Override
+ public List getRunningWorkflowIds(String workflowName, int version) {
+ return primaryDAO.getRunningWorkflowIds(workflowName, version);
+ }
+
+ @Override
+ public List getPendingWorkflowsByType(String workflowName, int version) {
+ return primaryDAO.getPendingWorkflowsByType(workflowName, version);
+ }
+
+ @Override
+ public long getPendingWorkflowCount(String workflowName) {
+ return primaryDAO.getPendingWorkflowCount(workflowName);
+ }
+
+ @Override
+ public List getTasks(String taskType, String startKey, int count) {
+ // This method is only intended to show pending tasks
+ return primaryDAO.getTasks(taskType, startKey, count);
+ }
+
+ @Override
+ public long getInProgressTaskCount(String taskDefName) {
+ return primaryDAO.getInProgressTaskCount(taskDefName);
+ }
+
+ @Override
+ public boolean canSearchAcrossWorkflows() {
+ return true;
+ }
+
+ ////////////////////////////////////////////////////////////////////////
+ // Hybrid Mode //
+ ////////////////////////////////////////////////////////////////////////
+
+ @Override
+ public String updateWorkflow(WorkflowModel workflow) {
+ return metricsCollector
+ .getTimer("update_workflow_dao", "workflowName", workflow.getWorkflowName())
+ .record(
+ () -> {
+ workflow.setUpdatedTime(System.currentTimeMillis());
+ String id = primaryDAO.updateWorkflow(workflow);
+ queueForIndexing(workflow, false);
+ if (workflow.getStatus().isTerminal()) {
+ metricsCollector.recordWorkflowComplete(workflow);
+ }
+ return id;
+ });
+ }
+
+ @Override
+ public TaskModel getTask(String taskId) {
+ return metricsCollector
+ .getTimer("get_task_dao")
+ .record(
+ () -> {
+ TaskModel task = primaryDAO.getTask(taskId);
+ return task;
+ });
+ }
+
+ @Override
+ public List getTasks(List taskIds) {
+ return metricsCollector
+ .getTimer("get_tasks_dao")
+ .record(() -> primaryDAO.getTasks(taskIds));
+ }
+
+ @Override
+ public List getTasksForWorkflow(String workflowId) {
+ return metricsCollector
+ .getTimer("get_tasks_for_workflow_dao")
+ .record(
+ () -> {
+ List tasks = primaryDAO.getTasksForWorkflow(workflowId);
+ if (tasks == null || tasks.isEmpty()) {
+ tasks = archiveDAO.getWorkflow(workflowId, true).getTasks();
+ }
+ return tasks;
+ });
+ }
+
+ @Override
+ public String createWorkflow(WorkflowModel workflow) {
+ // UUID used are time based and we want to keep the created time of the UUID with the create
+ // time of workflow
+ // The reason is that the create time is used for partitioning and
+ // we want to be able to get the create time from workflow id
+ long time = TimeBasedUUIDGenerator.getDate(workflow.getWorkflowId());
+ workflow.setCreateTime(time);
+
+ return metricsCollector
+ .getTimer("create_workflow_dao", "workflowName", workflow.getWorkflowName())
+ .record(
+ () -> {
+ workflow.setUpdatedTime(System.currentTimeMillis());
+ String workflowId = primaryDAO.createWorkflow(workflow);
+ queueForIndexing(workflow, true);
+ return workflowId;
+ });
+ }
+
+ @Override
+ public boolean removeWorkflow(String workflowId) {
+ boolean removed = primaryDAO.removeWorkflow(workflowId);
+ if (!removed) {
+ removed = archiveDAO.removeWorkflow(workflowId);
+ }
+ return removed;
+ }
+
+ @Override
+ public boolean removeWorkflowWithExpiry(String workflowId, int ttlSeconds) {
+ return primaryDAO.removeWorkflowWithExpiry(workflowId, ttlSeconds);
+ }
+
+ @Override
+ public WorkflowModel getWorkflow(String workflowId) {
+ return getWorkflow(workflowId, false);
+ }
+
+ @Override
+ public WorkflowModel getWorkflow(String workflowId, boolean includeTasks) {
+ WorkflowModel workflow = primaryDAO.getWorkflow(workflowId, includeTasks);
+ if (workflow == null) {
+ log.debug("Not found in primary dao, going to archive {}", workflowId);
+ workflow =
+ metricsCollector
+ .getTimer("get_workflow_archive_dao", "includeTasks", "" + includeTasks)
+ .record(() -> archiveDAO.getWorkflow(workflowId, includeTasks));
+ }
+ return workflow;
+ }
+
+ @Override
+ public List getWorkflowsByType(
+ String workflowName, Long startTime, Long endTime) {
+ List workflows = new ArrayList<>();
+ List workflowIds =
+ archiveDAO.getWorkflowIdsByType(workflowName, startTime, endTime);
+ for (String workflowId : workflowIds) {
+ workflows.add(getWorkflow(workflowId));
+ }
+
+ return workflows;
+ }
+
+ @Override
+ public List getWorkflowsByCorrelationId(
+ String workflowName, String correlationId, boolean includeTasks) {
+ List ids =
+ archiveDAO.getWorkflowIdsByCorrelationId(
+ workflowName, correlationId, false, includeTasks);
+ return ids.stream()
+ .map(id -> getWorkflow(id, includeTasks))
+ .filter(wf -> wf != null)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public boolean addEventExecution(EventExecution eventExecution) {
+ boolean added = primaryDAO.addEventExecution(eventExecution);
+ return added;
+ }
+
+ @Override
+ public void updateEventExecution(EventExecution eventExecution) {
+ primaryDAO.updateEventExecution(eventExecution);
+ }
+
+ @Override
+ public void removeEventExecution(EventExecution eventExecution) {
+ primaryDAO.removeEventExecution(eventExecution);
+ }
+
+ private void queueForIndexing(WorkflowModel workflow, boolean created) {
+
+ if (!created && !workflow.getStatus().isTerminal()) {
+ // Do nothing! We only index the workflow once its created and once its completed
+ return;
+ }
+ String messageId =
+ IndexWorker.WORKFLOW_ID_PREFIX
+ + workflow.getWorkflowId()
+ + ":"
+ + workflow.getStatus();
+ long offsetTime = OFFSET_TIME_SEC;
+
+ if (workflow.getStatus().isTerminal()) {
+ // Move ahead of the queue
+
+ // Below is how the score is calculated for pushing the message to the sorted set
+ // double score = Long.valueOf(clock.millis() + message.getTimeout()).doubleValue() +
+ // priority;
+
+ // Making the time to be negative pushes the message at the beginning of the queue
+ // Reducing the current time by 1s second, to ensure any mismatches do not cause score
+ // to be negative
+ // Negative score is allowed,but when querying the messages, the min score is set to 0
+ offsetTime = -1 * ((clock.millis() / 1000) - 1000);
+ }
+
+ if (!created && !workflow.getStatus().isTerminal()) {
+ // If this is not a newly created workflow and is not yet completed,
+ // We add a random delay to index
+ // Adding a delay ensures two things:
+ // 1. If the workflow completes in the next 1-2 seconds, the completed status will
+ // remove the pending
+ // workflow indexing --> see the block below
+ // 2. Probabiliy that multiple arallel threads/workers picking up the same workflow Id
+ // reduces
+ // avoiding database row lock contention
+
+ int delayInSeconds = Math.max(1, new Random().nextInt(10));
+ offsetTime = delayInSeconds;
+ }
+
+ queueDAO.push(INDEXER_QUEUE, messageId, offsetTime);
+
+ if (workflow.getStatus().isTerminal()) {
+ // Remove any previous message, so we can avoid indexing it twice
+ messageId =
+ IndexWorker.WORKFLOW_ID_PREFIX
+ + workflow.getWorkflowId()
+ + ":"
+ + Workflow.WorkflowStatus.RUNNING.toString();
+ queueDAO.ack(INDEXER_QUEUE, messageId);
+ }
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/archive/ArchivedIndexDAO.java b/archive/src/main/java/io/orkes/conductor/dao/archive/ArchivedIndexDAO.java
new file mode 100644
index 0000000..6a2ac24
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/archive/ArchivedIndexDAO.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.archive;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.common.metadata.events.EventExecution;
+import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
+import com.netflix.conductor.common.run.SearchResult;
+import com.netflix.conductor.common.run.TaskSummary;
+import com.netflix.conductor.common.run.WorkflowSummary;
+import com.netflix.conductor.core.events.queue.Message;
+import com.netflix.conductor.dao.IndexDAO;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Component
+@ConditionalOnProperty(name = "conductor.archive.db.enabled", havingValue = "true")
+@Primary
+@Slf4j
+public class ArchivedIndexDAO implements IndexDAO {
+
+ private final ArchiveDAO archiveDAO;
+
+ public ArchivedIndexDAO(ArchiveDAO archiveDAO) {
+ this.archiveDAO = archiveDAO;
+ }
+
+ @Override
+ public void setup() throws Exception {}
+
+ @Override
+ public void indexWorkflow(WorkflowSummary workflow) {}
+
+ @Override
+ public CompletableFuture asyncIndexWorkflow(WorkflowSummary workflow) {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public void indexTask(TaskSummary task) {
+ return;
+ }
+
+ @Override
+ public CompletableFuture asyncIndexTask(TaskSummary task) {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public SearchResult searchWorkflows(
+ String query, String freeText, int start, int count, List sort) {
+ return archiveDAO.searchWorkflows(query, freeText, start, count);
+ }
+
+ @Override
+ public SearchResult searchTasks(
+ String query, String freeText, int start, int count, List sort) {
+ throw new UnsupportedOperationException("Task search is not supported in this environment");
+ }
+
+ @Override
+ public void removeWorkflow(String workflowId) {
+ archiveDAO.removeWorkflow(workflowId);
+ }
+
+ @Override
+ public CompletableFuture asyncRemoveWorkflow(String workflowId) {
+ archiveDAO.removeWorkflow(workflowId);
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {}
+
+ @Override
+ public CompletableFuture asyncUpdateWorkflow(
+ String workflowInstanceId, String[] keys, Object[] values) {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public String get(String workflowInstanceId, String key) {
+ return null;
+ }
+
+ @Override
+ public void addTaskExecutionLogs(List logs) {
+ archiveDAO.addTaskExecutionLogs(logs);
+ }
+
+ @Override
+ public CompletableFuture asyncAddTaskExecutionLogs(List logs) {
+ archiveDAO.addTaskExecutionLogs(logs);
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public List getTaskExecutionLogs(String taskId) {
+ return archiveDAO.getTaskExecutionLogs(taskId);
+ }
+
+ @Override
+ public void addEventExecution(EventExecution eventExecution) {}
+
+ @Override
+ public List getEventExecutions(String event) {
+ return null;
+ }
+
+ @Override
+ public CompletableFuture asyncAddEventExecution(EventExecution eventExecution) {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public void addMessage(String queue, Message msg) {}
+
+ @Override
+ public CompletableFuture asyncAddMessage(String queue, Message message) {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ @Override
+ public List getMessages(String queue) {
+ return null;
+ }
+
+ @Override
+ public List searchArchivableWorkflows(String indexName, long archiveTtlDays) {
+ throw new UnsupportedOperationException("You do not need to use this! :)");
+ }
+
+ public long getWorkflowCount(String query, String freeText) {
+ return 0;
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/archive/DocumentStoreDAO.java b/archive/src/main/java/io/orkes/conductor/dao/archive/DocumentStoreDAO.java
new file mode 100644
index 0000000..4c45f17
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/archive/DocumentStoreDAO.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.archive;
+
+import com.netflix.conductor.model.WorkflowModel;
+
+/**
+ * Document store is used to store completed workflow JSON data.
+ *
+ * @see io.orkes.conductor.dao.postgres.archive.PostgresArchiveDAO
+ */
+public interface DocumentStoreDAO {
+
+ void createOrUpdateWorkflow(WorkflowModel workflow);
+
+ boolean removeWorkflow(String workflowId);
+
+ WorkflowModel getWorkflow(String workflowId, boolean includeTasks);
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/archive/ScrollableSearchResult.java b/archive/src/main/java/io/orkes/conductor/dao/archive/ScrollableSearchResult.java
new file mode 100644
index 0000000..c0dfe6b
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/archive/ScrollableSearchResult.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.archive;
+
+import java.util.List;
+
+import com.netflix.conductor.common.run.SearchResult;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@NoArgsConstructor
+public class ScrollableSearchResult extends SearchResult {
+
+ private String queryId;
+
+ public ScrollableSearchResult(List results, String queryId) {
+ super(0, results);
+ this.queryId = queryId;
+ }
+
+ // With ScrollableSearchResult this will always be zero and it's confusing from an API client's
+ // perspective.
+ // That's why it's ignored.
+ @Override
+ @JsonIgnore
+ public long getTotalHits() {
+ return super.getTotalHits();
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexValueExtractor.java b/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexValueExtractor.java
new file mode 100644
index 0000000..e2518bc
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexValueExtractor.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.indexer;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class IndexValueExtractor {
+
+ private static final String splitWords = "\"|,|;|\\s|,";
+ private static final String replaceWords = "\"|,|;|\\s|,|:";
+ private static final Collection stopWords = EnglishAnalyzer.getDefaultStopSet();
+
+ public static Collection getIndexWords(
+ WorkflowModel workflow, int maxWords, int maxWordLength) {
+
+ try {
+
+ List words = getIndexWords(workflow);
+ return words.stream()
+ .flatMap(value -> Arrays.asList(value.split(splitWords)).stream())
+ .filter(word -> word.length() < maxWordLength)
+ .filter(word -> word.length() > 2)
+ .filter(word -> !word.trim().isBlank())
+ .map(word -> word.toLowerCase().trim().replaceAll(replaceWords + "+$", ""))
+ .filter(word -> !stopWords.contains(word))
+ .limit(maxWords)
+ .collect(Collectors.toSet());
+
+ } catch (Exception e) {
+ log.warn("Error serializing input/output map to text: " + e.getMessage(), e);
+ return new ArrayList<>();
+ }
+ }
+
+ private static List getIndexWords(WorkflowModel workflow) {
+ List words = new ArrayList<>();
+ append(words, workflow.getCorrelationId());
+ append(words, workflow.getInput());
+ append(words, workflow.getOutput());
+ append(words, workflow.getReasonForIncompletion());
+ append(words, workflow.getVariables());
+
+ for (TaskModel task : workflow.getTasks()) {
+ append(words, task.getOutputData());
+ }
+ return words;
+ }
+
+ private static void append(List words, Object value) {
+ if (value instanceof String || value instanceof Number) {
+ if (value != null) words.add(value.toString());
+ } else if (value instanceof List) {
+ List values = (List) value;
+ for (Object valueObj : values) {
+ if (valueObj != null) words.add(valueObj.toString());
+ }
+ } else if (value instanceof Map) {
+ append(words, (Map) value);
+ }
+ }
+
+ private static void append(List words, Map map) {
+
+ map.values()
+ .forEach(
+ value -> {
+ if (value instanceof String || value instanceof Number) {
+ if (value != null) words.add(value.toString());
+ } else if (value instanceof Map) {
+ append(words, (Map) value);
+ } else if (value instanceof List) {
+ List values = (List) value;
+ for (Object valueObj : values) {
+ if (valueObj != null) words.add(valueObj.toString());
+ }
+ }
+ });
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorker.java b/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorker.java
new file mode 100644
index 0000000..9e0c45e
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorker.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.indexer;
+
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.dao.ExecutionDAO;
+import com.netflix.conductor.dao.QueueDAO;
+import com.netflix.conductor.metrics.Monitors;
+import com.netflix.conductor.model.WorkflowModel;
+
+import io.orkes.conductor.dao.archive.ArchiveDAO;
+import io.orkes.conductor.metrics.MetricsCollector;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@ConditionalOnProperty(name = "conductor.archive.db.enabled", havingValue = "true")
+public class IndexWorker {
+
+ public static final String INDEXER_QUEUE = "_index_queue";
+
+ public static final String WORKFLOW_ID_PREFIX = "wf:";
+
+ public static final String EVENT_ID_PREFIX = "e:";
+
+ private final QueueDAO queueDAO;
+
+ private final ArchiveDAO archiveDAO;
+
+ private final ExecutionDAO primaryExecDAO;
+
+ private final MetricsCollector metricsCollector;
+
+ private ScheduledExecutorService executorService;
+
+ private int pollBatchSize;
+
+ private int ttlSeconds = 30;
+
+ public IndexWorker(
+ QueueDAO queueDAO,
+ ArchiveDAO execDAO,
+ ExecutionDAO primaryExecDAO,
+ IndexWorkerProperties properties,
+ MetricsCollector metricsCollector) {
+ this.queueDAO = queueDAO;
+ this.archiveDAO = execDAO;
+ this.primaryExecDAO = primaryExecDAO;
+ this.pollBatchSize = properties.getPollBatchSize();
+ this.metricsCollector = metricsCollector;
+
+ int threadCount = properties.getThreadCount();
+ int pollingInterval = properties.getPollingInterval();
+ if (threadCount > 0) {
+ this.executorService =
+ Executors.newScheduledThreadPool(
+ threadCount,
+ new ThreadFactoryBuilder().setNameFormat("indexer-thread-%d").build());
+
+ for (int i = 0; i < threadCount; i++) {
+ this.executorService.scheduleWithFixedDelay(
+ () -> {
+ try {
+ pollAndExecute();
+ } catch (Throwable t) {
+ log.error(t.getMessage(), t);
+ }
+ },
+ 10,
+ pollingInterval,
+ TimeUnit.MILLISECONDS);
+ }
+ }
+
+ log.info(
+ "IndexWorker::INIT with primaryExecDAO = {}, threadCount = {}, pollingInterval = {} and pollBatchSize = {}",
+ primaryExecDAO,
+ threadCount,
+ pollingInterval,
+ pollBatchSize);
+ }
+
+ private void pollAndExecute() {
+
+ try {
+
+ List ids = queueDAO.pop(INDEXER_QUEUE, pollBatchSize, 1000); // 1 second
+
+ for (String id : ids) {
+ if (id.startsWith(WORKFLOW_ID_PREFIX)) {
+ String workflowId = id.substring(WORKFLOW_ID_PREFIX.length());
+ workflowId = workflowId.substring(0, workflowId.lastIndexOf(':'));
+ indexWorkflow(workflowId);
+ queueDAO.ack(INDEXER_QUEUE, id);
+ } else if (id.startsWith(EVENT_ID_PREFIX)) {
+ String eventId = id.substring(EVENT_ID_PREFIX.length());
+ eventId = eventId.substring(0, eventId.lastIndexOf(':'));
+ indexEvent(eventId);
+ queueDAO.ack(INDEXER_QUEUE, id);
+ }
+ }
+
+ } catch (Throwable e) {
+ Monitors.error(IndexWorker.class.getSimpleName(), "pollAndExecute");
+ log.error(e.getMessage(), e);
+ } finally {
+ Monitors.recordQueueDepth(INDEXER_QUEUE, queueDAO.getSize(INDEXER_QUEUE), "");
+ }
+ }
+
+ private void indexWorkflow(String workflowId) {
+ WorkflowModel workflow = primaryExecDAO.getWorkflow(workflowId, true);
+ if (workflow == null) {
+ log.warn("Cannot find workflow in the primary DAO: {}", workflowId);
+ return;
+ }
+
+ metricsCollector
+ .getTimer("archive_workflow", "workflowName", "" + workflow.getWorkflowName())
+ .record(() -> archiveDAO.createOrUpdateWorkflow(workflow));
+
+ if (workflow.getStatus().isTerminal()) {
+ primaryExecDAO.removeWorkflowWithExpiry(workflowId, ttlSeconds);
+ }
+ }
+
+ private void indexEvent(String eventId) {
+ log.trace("Indexing event {}", eventId);
+ // Do nothing for now
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorkerProperties.java b/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorkerProperties.java
new file mode 100644
index 0000000..7811821
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/indexer/IndexWorkerProperties.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.indexer;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@ConfigurationProperties("conductor.archive.db.indexer")
+public class IndexWorkerProperties {
+
+ private int threadCount = 3;
+
+ private int pollingInterval = 10; // in millisecond
+
+ private int pollBatchSize = 10;
+
+ public int getThreadCount() {
+ return threadCount;
+ }
+
+ public void setThreadCount(int threadCount) {
+ this.threadCount = threadCount;
+ }
+
+ public int getPollingInterval() {
+ return pollingInterval;
+ }
+
+ public void setPollingInterval(int pollingInterval) {
+ this.pollingInterval = pollingInterval;
+ }
+
+ public int getPollBatchSize() {
+ return pollBatchSize;
+ }
+
+ public void setPollBatchSize(int pollBatchSize) {
+ this.pollBatchSize = pollBatchSize;
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/indexer/TaskIndex.java b/archive/src/main/java/io/orkes/conductor/dao/indexer/TaskIndex.java
new file mode 100644
index 0000000..4395a14
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/indexer/TaskIndex.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.indexer;
+
+import java.util.List;
+import java.util.Map;
+
+import com.netflix.conductor.model.TaskModel;
+
+public class TaskIndex {
+
+ private TaskModel task;
+
+ private int maxValueLen;
+
+ public TaskIndex(TaskModel task, int maxValueLen) {
+ this.task = task;
+ this.maxValueLen = maxValueLen;
+ }
+
+ public String toIndexString() {
+ StringBuilder sb = new StringBuilder();
+ append(sb, task.getTaskType());
+ append(sb, task.getTaskId());
+ append(sb, task.getDomain());
+ append(sb, task.getReferenceTaskName());
+ append(sb, task.getStatus().toString());
+ append(sb, task.getExternalInputPayloadStoragePath());
+ append(sb, task.getExternalOutputPayloadStoragePath());
+ append(sb, task.getInputData());
+ append(sb, task.getOutputData());
+ append(sb, task.getReasonForIncompletion());
+ append(sb, task.getWorkerId());
+
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return toIndexString();
+ }
+
+ private String toString(Map map) {
+ StringBuilder sb = new StringBuilder();
+ append(sb, map);
+ return sb.toString();
+ }
+
+ private void append(StringBuilder sb, Object value) {
+ if (value instanceof String || value instanceof Number) {
+ sb.append(" ");
+ sb.append(value.toString());
+ } else if (value instanceof Map) {
+ sb.append(" ");
+ append(sb, (Map) value);
+ } else if (value instanceof List) {
+ List values = (List) value;
+ for (Object valueObj : values) {
+ sb.append(" ");
+ append(sb, valueObj);
+ }
+ }
+ }
+
+ private void append(StringBuilder sb, Map map) {
+
+ map.entrySet()
+ .forEach(
+ entry -> {
+ String key = entry.getKey();
+ Object value = entry.getValue();
+ sb.append(" ");
+ sb.append(key);
+ if (value instanceof String) {
+ sb.append(" ");
+ String valueString = value.toString();
+ sb.append(
+ valueString.substring(
+ 0, Math.min(valueString.length(), maxValueLen)));
+ } else if (value instanceof Number) {
+ sb.append(" ");
+ sb.append(value.toString());
+ } else if (value instanceof Map) {
+ sb.append(" ");
+ append(sb, (Map) value);
+ } else if (value instanceof List) {
+ List values = (List) value;
+ for (Object valueObj : values) {
+ sb.append(" ");
+ append(sb, valueObj);
+ }
+ }
+ });
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/indexer/WorkflowIndex.java b/archive/src/main/java/io/orkes/conductor/dao/indexer/WorkflowIndex.java
new file mode 100644
index 0000000..32efcfb
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/indexer/WorkflowIndex.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.indexer;
+
+import java.util.Collection;
+
+import com.netflix.conductor.model.WorkflowModel;
+
+public class WorkflowIndex {
+
+ private WorkflowModel workflow;
+ private int maxWords;
+ private int maxWordLength;
+
+ public WorkflowIndex(WorkflowModel workflow, int maxWords, int maxWordLength) {
+ this.workflow = workflow;
+ this.maxWords = maxWords;
+ this.maxWordLength = maxWordLength;
+ }
+
+ public Collection toIndexWords() {
+ return IndexValueExtractor.getIndexWords(workflow, maxWords, maxWordLength);
+ }
+
+ @Override
+ public String toString() {
+ return toIndexWords().toString();
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/PostgresArchiveDAO.java b/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/PostgresArchiveDAO.java
new file mode 100644
index 0000000..1c4624e
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/PostgresArchiveDAO.java
@@ -0,0 +1,417 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres.archive;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import javax.sql.DataSource;
+
+import org.apache.logging.log4j.util.Strings;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.retry.support.RetryTemplate;
+
+import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
+import com.netflix.conductor.common.run.SearchResult;
+import com.netflix.conductor.model.WorkflowModel;
+import com.netflix.conductor.postgres.dao.PostgresBaseDAO;
+
+import io.orkes.conductor.dao.archive.ArchiveDAO;
+import io.orkes.conductor.dao.archive.DocumentStoreDAO;
+import io.orkes.conductor.dao.archive.ScrollableSearchResult;
+import io.orkes.conductor.dao.indexer.WorkflowIndex;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class PostgresArchiveDAO extends PostgresBaseDAO implements ArchiveDAO, DocumentStoreDAO {
+
+ private static final String GET_WORKFLOW =
+ "SELECT json_data FROM workflow_archive WHERE workflow_id = ? FOR SHARE SKIP LOCKED";
+
+ private static final String REMOVE_WORKFLOW =
+ "DELETE FROM workflow_archive WHERE workflow_id = ?";
+
+ private final DataSource searchDatasource;
+
+ private static final String TABLE_NAME = "workflow_archive";
+
+ public PostgresArchiveDAO(
+ ObjectMapper objectMapper,
+ DataSource dataSource,
+ @Qualifier("searchDatasource") DataSource searchDatasource) {
+ super(RetryTemplate.defaultInstance(), objectMapper, dataSource);
+ this.searchDatasource = searchDatasource;
+
+ log.info("Using {} as search datasource", searchDatasource);
+
+ try (Connection conn = searchDatasource.getConnection()) {
+ log.info("Using {} as search datasource", conn.getMetaData().getURL());
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ }
+ }
+
+ public void createOrUpdateWorkflow(WorkflowModel workflow) {
+
+ String INSERT_OR_UPDATE_LATEST =
+ "INSERT INTO "
+ + TABLE_NAME
+ + " as wf"
+ + "(workflow_id, created_on, modified_on, correlation_id, workflow_name, status, index_data, created_by, json_data) "
+ + "VALUES "
+ + "(?, ?, ?, ?, ?, ?, ?, ?, ?) "
+ + "ON CONFLICT (workflow_id) DO "
+ + "UPDATE SET modified_on = ?, status = ?, index_data = ?, json_data = ? "
+ + "WHERE wf.modified_on < ? ;";
+
+ try (Connection connection = super.dataSource.getConnection()) {
+ connection.setAutoCommit(true);
+ PreparedStatement statement = connection.prepareStatement(INSERT_OR_UPDATE_LATEST);
+
+ WorkflowIndex index = new WorkflowIndex(workflow, 200, 50);
+ Collection indexData = index.toIndexWords();
+
+ int indx = 1;
+ long updatedTime = workflow.getUpdatedTime() == null ? 0 : workflow.getUpdatedTime();
+
+ // Insert values
+
+ statement.setString(indx++, workflow.getWorkflowId());
+ statement.setLong(indx++, workflow.getCreateTime());
+ statement.setLong(indx++, updatedTime);
+ statement.setString(indx++, workflow.getCorrelationId());
+ statement.setString(indx++, workflow.getWorkflowName());
+ statement.setString(indx++, workflow.getStatus().toString());
+ statement.setArray(
+ indx++, connection.createArrayOf("text", indexData.toArray(new String[0])));
+ statement.setString(indx++, workflow.getCreatedBy());
+
+ String workflowJson = null;
+ if (workflow.getStatus().isTerminal()) {
+ workflowJson = objectMapper.writeValueAsString(workflow);
+ }
+ statement.setString(indx++, workflowJson);
+ // Update values
+ statement.setLong(indx++, updatedTime);
+ statement.setString(indx++, workflow.getStatus().toString());
+ statement.setArray(
+ indx++, connection.createArrayOf("text", indexData.toArray(new String[0])));
+ statement.setString(indx++, workflowJson);
+ statement.setLong(indx, updatedTime);
+
+ statement.executeUpdate();
+
+ } catch (Exception e) {
+ log.error(
+ "Error updating workflow {} - {}", workflow.getWorkflowId(), e.getMessage(), e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean removeWorkflow(String workflowId) {
+ boolean removed = false;
+ WorkflowModel workflow = this.getWorkflow(workflowId, true);
+ if (workflow != null) {
+ withTransaction(
+ connection ->
+ execute(
+ connection,
+ REMOVE_WORKFLOW,
+ q -> q.addParameter(workflowId).executeDelete()));
+ removed = true;
+ }
+ return removed;
+ }
+
+ @Override
+ public WorkflowModel getWorkflow(String workflowId, boolean includeTasks) {
+ try (Connection connection = super.dataSource.getConnection()) {
+ PreparedStatement statement = connection.prepareStatement(GET_WORKFLOW);
+ statement.setString(1, workflowId);
+ ResultSet rs = statement.executeQuery();
+ if (rs.next()) {
+ byte[] json = rs.getBytes("json_data");
+ if (json == null || json.length == 0) {
+ return null;
+ }
+ return objectMapper.readValue(json, WorkflowModel.class);
+ }
+ } catch (Exception e) {
+ log.error("Error reading workflow - " + e.getMessage(), e);
+ throw new RuntimeException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public List getWorkflowIdsByType(String workflowName, Long startTime, Long endTime) {
+ String query =
+ "workflowType IN ("
+ + workflowName
+ + ") AND startTime>"
+ + startTime
+ + " AND startTime< "
+ + endTime;
+ ScrollableSearchResult result = searchWorkflows(query, "*", 0, 100_000);
+ return result.getResults();
+ }
+
+ @Override
+ public List getWorkflowIdsByCorrelationId(
+ String workflowName,
+ String correlationId,
+ boolean includeClosed,
+ boolean includeTasks) {
+ String query =
+ "workflowType = '"
+ + workflowName.trim()
+ + "' AND correlationId = '"
+ + correlationId.trim()
+ + "'";
+ if (!includeClosed) {
+ query += " AND status IN (" + WorkflowModel.Status.RUNNING + ")";
+ }
+ ScrollableSearchResult result = searchWorkflows(query, null, 0, 100_000);
+ return result.getResults();
+ }
+
+ // Search
+ public ScrollableSearchResult searchWorkflows(
+ String query, String freeText, int start, int count) {
+
+ if (query == null) query = "";
+ if (freeText == null) freeText = "";
+
+ log.debug(
+ "search. query = {}, fulltext={}, limit={}, start= {}",
+ query,
+ freeText,
+ count,
+ start);
+
+ SearchQuery parsedQuery = SearchQuery.parse(query);
+ SearchResult results =
+ search(TABLE_NAME, parsedQuery, freeText.trim(), count, start);
+ ScrollableSearchResult scrollableSearchResult = new ScrollableSearchResult<>();
+ scrollableSearchResult.setResults(results.getResults());
+ return scrollableSearchResult;
+ }
+
+ private SearchResult search(
+ String tableName, SearchQuery query, String freeText, int limit, int start) {
+
+ List workflowNames = query.getWorkflowNames();
+ List workflowIds = query.getWorkflowIds();
+ List correlationIds = query.getCorrelationIds();
+ List statuses = query.getStatuses();
+ long startTime = query.getFromStartTime();
+ long endTime = query.getToStartTime();
+ if (endTime == 0) {
+ endTime = System.currentTimeMillis();
+ }
+
+ boolean search = false;
+
+ // Task specific
+ List taskIds = query.getTaskIds();
+ List taskTypes = query.getTaskTypes();
+
+ String WHERE_CLAUSE = "from " + tableName + " archive ";
+
+ WHERE_CLAUSE += " WHERE 1=1 ";
+
+ String SELECT_QUERY = "select workflow_id, created_on ";
+
+ String JOINER = " AND ";
+ if (workflowNames != null && !workflowNames.isEmpty()) {
+ WHERE_CLAUSE += JOINER + "archive.workflow_name = ANY (?) ";
+ }
+
+ if (taskTypes != null && !taskTypes.isEmpty()) {
+ WHERE_CLAUSE += JOINER + "task_type = ANY (?) ";
+ }
+
+ if (workflowIds != null && !workflowIds.isEmpty()) {
+ WHERE_CLAUSE += JOINER + "workflow_id = ANY (?) ";
+ JOINER = " AND ";
+ }
+
+ if (taskIds != null && !taskIds.isEmpty()) {
+ WHERE_CLAUSE += JOINER + "task_id = ANY (?) ";
+ JOINER = " AND ";
+ }
+
+ if (statuses != null && !statuses.isEmpty()) {
+ WHERE_CLAUSE += JOINER + "status = ANY (?) ";
+ JOINER = " AND ";
+ }
+
+ if (correlationIds != null && !correlationIds.isEmpty()) {
+ WHERE_CLAUSE += JOINER + "correlation_id = ANY (?) ";
+ JOINER = " AND ";
+ }
+
+ if (startTime > 0) {
+ WHERE_CLAUSE += JOINER + "created_on BETWEEN ? AND ? ";
+ JOINER = " AND ";
+ }
+
+ if (Strings.isNotBlank(freeText) && !"*".equals(freeText)) {
+ search = true;
+ WHERE_CLAUSE += JOINER + " index_data @> ? ";
+ }
+
+ String SEARCH_QUERY =
+ SELECT_QUERY
+ + " "
+ + WHERE_CLAUSE
+ + " order by created_on desc limit "
+ + limit
+ + " offset "
+ + start;
+ if (search) {
+ SEARCH_QUERY =
+ "select a.workflow_id, a.created_on from ("
+ + SELECT_QUERY
+ + " "
+ + WHERE_CLAUSE
+ + " limit 2000000 offset 0) a order by a.created_on desc limit "
+ + limit
+ + " offset "
+ + start;
+ }
+
+ log.debug(SEARCH_QUERY);
+
+ SearchResult result = new SearchResult<>();
+ result.setResults(new ArrayList<>());
+
+ try (Connection conn = searchDatasource.getConnection()) {
+ PreparedStatement pstmt = conn.prepareStatement(SEARCH_QUERY);
+ int indx = 1;
+ if (workflowNames != null && !workflowNames.isEmpty()) {
+ pstmt.setArray(
+ indx++,
+ conn.createArrayOf("VARCHAR", workflowNames.toArray(new String[0])));
+ }
+ if (taskTypes != null && !taskTypes.isEmpty()) {
+ pstmt.setArray(
+ indx++, conn.createArrayOf("VARCHAR", taskTypes.toArray(new String[0])));
+ }
+
+ if (workflowIds != null && !workflowIds.isEmpty()) {
+ pstmt.setArray(
+ indx++, conn.createArrayOf("VARCHAR", workflowIds.toArray(new String[0])));
+ }
+
+ if (taskIds != null && !taskIds.isEmpty()) {
+ pstmt.setArray(
+ indx++, conn.createArrayOf("VARCHAR", taskIds.toArray(new String[0])));
+ }
+
+ if (statuses != null && !statuses.isEmpty()) {
+ pstmt.setArray(
+ indx++, conn.createArrayOf("VARCHAR", statuses.toArray(new String[0])));
+ }
+
+ if (correlationIds != null && !correlationIds.isEmpty()) {
+ pstmt.setArray(
+ indx++,
+ conn.createArrayOf("VARCHAR", correlationIds.toArray(new String[0])));
+ }
+
+ if (startTime > 0) {
+ pstmt.setLong(indx++, startTime);
+ pstmt.setLong(indx++, endTime);
+ }
+
+ if (Strings.isNotBlank(freeText) && !"*".equals(freeText)) {
+ String[] textArray = freeText.toLowerCase().split(" ");
+ pstmt.setArray(indx++, conn.createArrayOf("text", textArray));
+ }
+
+ result.setTotalHits(0);
+ long countStart = System.currentTimeMillis();
+ ResultSet rs = pstmt.executeQuery();
+ log.debug(
+ "search query took {} ms to execute",
+ (System.currentTimeMillis() - countStart));
+ while (rs.next()) {
+ String workflowId = rs.getString("workflow_id");
+ result.getResults().add(workflowId);
+ }
+
+ } catch (SQLException sqlException) {
+ log.error(sqlException.getMessage(), sqlException);
+ throw new RuntimeException(sqlException);
+ }
+
+ return result;
+ }
+
+ // Private Methods
+ @Override
+ public List getTaskExecutionLogs(String taskId) {
+ String GET_TASK = "SELECT seq, log, created_on FROM task_logs WHERE task_id = ?";
+ return queryWithTransaction(
+ GET_TASK,
+ q -> {
+ List taskExecLogs =
+ q.addParameter(taskId)
+ .executeAndFetch(
+ resultSet -> {
+ List logs = new ArrayList<>();
+ while (resultSet.next()) {
+ TaskExecLog log = new TaskExecLog();
+ log.setTaskId(taskId);
+ log.setLog(resultSet.getString(2));
+ log.setCreatedTime(resultSet.getLong(3));
+ logs.add(log);
+ }
+ return logs;
+ });
+ return taskExecLogs;
+ });
+ }
+
+ @Override
+ public void addTaskExecutionLogs(List logs) {
+ String INSERT_STMT = "INSERT INTO task_logs (task_id, log, created_on) values(?,?,?)";
+ for (TaskExecLog taskExecLog : logs) {
+ withTransaction(
+ tx -> {
+ execute(
+ tx,
+ INSERT_STMT,
+ q ->
+ q
+
+ // Insert values
+ .addParameter(taskExecLog.getTaskId())
+ .addParameter(taskExecLog.getLog())
+ .addParameter(taskExecLog.getCreatedTime())
+
+ // execute
+ .executeUpdate());
+ });
+ }
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/PostgresArchiveDAOConfiguration.java b/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/PostgresArchiveDAOConfiguration.java
new file mode 100644
index 0000000..409ec2c
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/PostgresArchiveDAOConfiguration.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres.archive;
+
+import javax.annotation.PostConstruct;
+import javax.sql.DataSource;
+
+import org.apache.logging.log4j.util.Strings;
+import org.flywaydb.core.Flyway;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.autoconfigure.flyway.FlywayConfigurationCustomizer;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.*;
+import org.springframework.core.env.Environment;
+
+import com.netflix.conductor.dao.ExecutionDAO;
+import com.netflix.conductor.dao.QueueDAO;
+import com.netflix.conductor.postgres.config.PostgresProperties;
+
+import io.orkes.conductor.dao.archive.ArchiveDAO;
+import io.orkes.conductor.dao.archive.ArchivedExecutionDAO;
+import io.orkes.conductor.metrics.MetricsCollector;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Configuration(proxyBeanMethods = false)
+@EnableConfigurationProperties({PostgresProperties.class})
+@Import(DataSourceAutoConfiguration.class)
+@ConditionalOnProperty(name = "conductor.archive.db.type", havingValue = "postgres")
+public class PostgresArchiveDAOConfiguration {
+
+ private final DataSource dataSource;
+
+ private final MetricsCollector metricsCollector;
+
+ private final ExecutionDAO primaryExecutionDAO;
+
+ private final QueueDAO dynoQueueDAO;
+
+ private final Environment environment;
+
+ private final ObjectMapper objectMapper;
+
+ public PostgresArchiveDAOConfiguration(
+ ObjectMapper objectMapper,
+ Environment environment,
+ DataSource dataSource,
+ ExecutionDAO primaryExecutionDAO,
+ QueueDAO dynoQueueDAO,
+ MetricsCollector metricsCollector) {
+
+ this.objectMapper = objectMapper;
+ this.environment = environment;
+ this.dataSource = dataSource;
+ this.primaryExecutionDAO = primaryExecutionDAO;
+ this.dynoQueueDAO = dynoQueueDAO;
+ this.metricsCollector = metricsCollector;
+ }
+
+ @Bean
+ @Primary
+ @ConditionalOnProperty(name = "conductor.archive.db.enabled", havingValue = "true")
+ public ExecutionDAO getExecutionDAO(ArchiveDAO archiveDAO) {
+ return new ArchivedExecutionDAO(
+ primaryExecutionDAO, archiveDAO, dynoQueueDAO, metricsCollector);
+ }
+
+ @Bean(initMethod = "migrate", name = "flyway")
+ @PostConstruct
+ public Flyway flywayForPrimaryDb() {
+ return Flyway.configure()
+ .locations(
+ "classpath:db/migration_postgres",
+ "classpath:db/migration_archive_postgres")
+ .schemas("public")
+ .dataSource(dataSource)
+ .baselineOnMigrate(true)
+ .mixed(true)
+ .load();
+ }
+
+ @Bean(name = "flywayInitializer")
+ public FlywayConfigurationCustomizer flywayConfigurationCustomizer() {
+ // override the default location.
+ return configuration ->
+ configuration.locations(
+ "classpath:db/migration_postgres",
+ "classpath:db/migration_archive_postgres");
+ }
+
+ @Bean
+ @Qualifier("searchDatasource")
+ public DataSource searchDatasource(DataSource defaultDatasource) {
+ String url = environment.getProperty("spring.search-datasource.url");
+ String user = environment.getProperty("spring.search-datasource.username");
+ String password = environment.getProperty("spring.search-datasource.password");
+ String maxPoolSizeString =
+ environment.getProperty("spring.search-datasource.hikari.maximum-pool-size");
+
+ if (Strings.isEmpty(url)) {
+ return defaultDatasource;
+ }
+ log.info("Configuring searchDatasource with {}", url);
+
+ int maxPoolSize = 10;
+ if (Strings.isNotEmpty(maxPoolSizeString)) {
+ try {
+ maxPoolSize = Integer.parseInt(maxPoolSizeString);
+ } catch (Exception e) {
+ }
+ }
+ HikariConfig config = new HikariConfig();
+ config.setJdbcUrl(url);
+ config.setAutoCommit(true);
+ config.setUsername(user);
+ config.setPassword(password);
+ config.setMaximumPoolSize(maxPoolSize);
+
+ HikariDataSource hikariDataSource = new HikariDataSource(config);
+ return hikariDataSource;
+ }
+
+ @Bean
+ @DependsOn({"flyway", "flywayInitializer"})
+ @ConditionalOnProperty(value = "conductor.archive.db.type", havingValue = "postgres")
+ public PostgresArchiveDAO getPostgresArchiveDAO(
+ @Qualifier("searchDatasource") DataSource searchDatasource) {
+ return new PostgresArchiveDAO(objectMapper, dataSource, searchDatasource);
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/SearchQuery.java b/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/SearchQuery.java
new file mode 100644
index 0000000..aafef30
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/dao/postgres/archive/SearchQuery.java
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres.archive;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class SearchQuery {
+
+ private List workflowNames;
+
+ private List taskTypes;
+
+ private List workflowIds;
+
+ private List correlationIds;
+
+ private List taskIds;
+
+ private List statuses;
+
+ private long fromStartTime;
+
+ private long toStartTime;
+
+ private SearchQuery() {}
+
+ public static SearchQuery parse(String query) {
+ SearchQuery searchQuery = new SearchQuery();
+
+ String[] values = query.split(" AND ");
+ for (String value : values) {
+ value = value.trim();
+ if (value.startsWith("workflowId")) {
+ searchQuery.workflowIds = getValues(value);
+ }
+ if (value.startsWith("correlationId")) {
+ searchQuery.correlationIds = getValues(value);
+ } else if (value.startsWith("taskId")) {
+ searchQuery.taskIds = getValues(value);
+ } else if (value.startsWith("workflowType")) {
+ searchQuery.workflowNames = getValues(value);
+ } else if (value.startsWith("taskType")) {
+ searchQuery.taskTypes = getValues(value);
+ } else if (value.startsWith("status")) {
+ searchQuery.statuses = getValues(value);
+ } else if (value.startsWith("startTime")) {
+
+ if (value.contains(">")) {
+
+ String[] kv = value.split(">");
+ if (kv.length > 0) {
+ try {
+ searchQuery.fromStartTime = Long.parseLong(kv[1].trim());
+ } catch (Exception e) {
+ }
+ }
+
+ } else if (value.contains("<")) {
+
+ String[] kv = value.split("<");
+ if (kv.length > 0) {
+ try {
+ searchQuery.toStartTime = Long.parseLong(kv[1].trim());
+ } catch (Exception e) {
+ }
+ }
+ }
+ }
+ }
+ return searchQuery;
+ }
+
+ private static List getValues(String keyValue) {
+ List values = new ArrayList<>();
+ if (keyValue.contains("=")) {
+ String[] kv = keyValue.split("=");
+ if (kv.length > 0) {
+ String value = kv[1].trim();
+ // remove quotes from the start and end
+ value = value.substring(1, value.length() - 1);
+ return Arrays.asList(value.trim());
+ }
+ } else if (keyValue.contains(" IN ")) {
+
+ String[] kv = keyValue.split(" IN ");
+ if (kv.length > 0) {
+ String[] inValues = kv[1].trim().substring(1, kv[1].length() - 1).split(",");
+ for (String inValue : inValues) {
+ values.add(inValue.trim());
+ }
+ }
+ }
+ return values;
+ }
+
+ public List getWorkflowNames() {
+ return workflowNames;
+ }
+
+ public List getWorkflowIds() {
+ return workflowIds;
+ }
+
+ public List getCorrelationIds() {
+ return correlationIds;
+ }
+
+ public List getStatuses() {
+ return statuses;
+ }
+
+ public long getFromStartTime() {
+ return fromStartTime;
+ }
+
+ public long getToStartTime() {
+ if (toStartTime == 0) {
+ toStartTime = System.currentTimeMillis();
+ }
+ return toStartTime;
+ }
+
+ public List getTaskTypes() {
+ return taskTypes;
+ }
+
+ public List getTaskIds() {
+ return taskIds;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ SearchQuery that = (SearchQuery) o;
+ return fromStartTime == that.fromStartTime
+ && toStartTime == that.toStartTime
+ && Objects.equals(workflowNames, that.workflowNames)
+ && Objects.equals(workflowIds, that.workflowIds)
+ && Objects.equals(statuses, that.statuses);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(workflowNames, workflowIds, statuses, fromStartTime, toStartTime);
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/id/TimeBasedUUIDGenerator.java b/archive/src/main/java/io/orkes/conductor/id/TimeBasedUUIDGenerator.java
new file mode 100644
index 0000000..eb27b38
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/id/TimeBasedUUIDGenerator.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.id;
+
+import java.time.LocalDate;
+import java.util.Calendar;
+import java.util.TimeZone;
+import java.util.UUID;
+
+import org.apache.logging.log4j.core.util.UuidUtil;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.core.utils.IDGenerator;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Component
+@ConditionalOnProperty(name = "conductor.id.generator", havingValue = "time_based")
+@Slf4j
+public class TimeBasedUUIDGenerator extends IDGenerator {
+
+ private static final LocalDate JAN_1_2020 = LocalDate.of(2020, 1, 1);
+
+ private static final int uuidLength = UUID.randomUUID().toString().length();
+
+ private static Calendar uuidEpoch = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+
+ private static final long epochMillis;
+
+ static {
+ uuidEpoch.clear();
+ uuidEpoch.set(1582, 9, 15, 0, 0, 0); //
+ epochMillis = uuidEpoch.getTime().getTime();
+ }
+
+ public TimeBasedUUIDGenerator() {
+ log.info("Using TimeBasedUUIDGenerator to generate Ids");
+ }
+
+ public String generate() {
+ UUID uuid = UuidUtil.getTimeBasedUuid();
+ return uuid.toString();
+ }
+
+ public static long getDate(String id) {
+ UUID uuid = UUID.fromString(id);
+ if (uuid.version() != 1) {
+ return 0;
+ }
+ long time = (uuid.timestamp() / 10000L) + epochMillis;
+ return time;
+ }
+}
diff --git a/archive/src/main/java/io/orkes/conductor/metrics/MetricsCollector.java b/archive/src/main/java/io/orkes/conductor/metrics/MetricsCollector.java
new file mode 100644
index 0000000..cd00980
--- /dev/null
+++ b/archive/src/main/java/io/orkes/conductor/metrics/MetricsCollector.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+import com.netflix.spectator.api.CompositeRegistry;
+import com.netflix.spectator.api.Spectator;
+import com.netflix.spectator.micrometer.MicrometerRegistry;
+
+import io.micrometer.core.instrument.*;
+import io.micrometer.core.instrument.config.MeterFilter;
+import io.micrometer.prometheus.PrometheusRenameFilter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+public class MetricsCollector {
+
+ private static final CompositeRegistry spectatorRegistry = Spectator.globalRegistry();
+
+ private final MeterRegistry meterRegistry;
+
+ private static final Map, Counter>> counters = new ConcurrentHashMap<>();
+
+ private static final Map, Timer>> timers = new ConcurrentHashMap<>();
+
+ private static final Map, Gauge>> gauges = new ConcurrentHashMap<>();
+
+ private static double[] percentiles = new double[] {0.5, 0.75, 0.90, 0.95, 0.99};
+
+ private final boolean skipLabels;
+
+ public MetricsCollector(MeterRegistry meterRegistry, Environment env) {
+
+ Boolean skipLabels = env.getProperty("conductor.metrics.skipLabels", Boolean.class);
+ if (skipLabels == null) {
+ this.skipLabels = false;
+ } else {
+ this.skipLabels = skipLabels;
+ }
+
+ this.meterRegistry = meterRegistry;
+ final MicrometerRegistry metricsRegistry = new MicrometerRegistry(this.meterRegistry);
+
+ this.meterRegistry
+ .config()
+ .meterFilter(new PrometheusRenameFilter())
+ .meterFilter(MeterFilter.denyNameStartsWith("task_queue_wait"))
+ .meterFilter(MeterFilter.denyNameStartsWith("dao_payload_size"))
+ .meterFilter(MeterFilter.denyNameStartsWith("dao_requests"))
+ .meterFilter(MeterFilter.denyNameStartsWith("workflow_execution"))
+ .meterFilter(MeterFilter.denyNameStartsWith("tasks_in_workflow"))
+ .meterFilter(MeterFilter.denyNameStartsWith("task_pending_time"))
+ .meterFilter(MeterFilter.denyNameStartsWith("workflow_start_success"))
+ .meterFilter(MeterFilter.denyNameStartsWith("task_execution"));
+
+ if (this.skipLabels) {
+ this.meterRegistry
+ .config()
+ .meterFilter(MeterFilter.denyNameStartsWith("workflow_failure"))
+ .meterFilter(MeterFilter.denyNameStartsWith("workflow_running"));
+ }
+
+ spectatorRegistry.add(metricsRegistry);
+
+ log.info("skipLabels: {}", this.skipLabels);
+ }
+
+ public Timer getTimer(String name, String... additionalTags) {
+ List tags = tags(additionalTags);
+ return timers.computeIfAbsent(name, s -> new ConcurrentHashMap<>())
+ .computeIfAbsent(
+ tags,
+ t -> {
+ Timer.Builder timerBuilder =
+ Timer.builder(name)
+ .description(name)
+ .publishPercentiles(percentiles);
+ if (!this.skipLabels) {
+ timerBuilder = timerBuilder.tags(tags);
+ }
+ return timerBuilder.register(meterRegistry);
+ });
+ }
+
+ public Counter getCounter(String name, String... additionalTags) {
+ List tags = tags(additionalTags);
+ return counters.computeIfAbsent(name, s -> new ConcurrentHashMap<>())
+ .computeIfAbsent(
+ tags,
+ t -> {
+ Counter.Builder builder = Counter.builder(name).description(name);
+ if (!this.skipLabels) {
+ builder = builder.tags(tags);
+ }
+ return builder.register(meterRegistry);
+ });
+ }
+
+ public void recordGauge(String name, Number value, String... additionalTags) {
+ List tags = tags(additionalTags);
+ gauges.computeIfAbsent(name, s -> new ConcurrentHashMap<>())
+ .computeIfAbsent(
+ tags,
+ t -> {
+ Gauge.Builder> builder =
+ Gauge.builder(name, () -> value);
+
+ if (!this.skipLabels) {
+ builder = builder.tags(tags);
+ }
+ return builder.register(meterRegistry);
+ });
+ }
+
+ private void gauge(String name, Number value, String... additionalTags) {
+ Gauge.builder(name, () -> value).register(meterRegistry);
+ }
+
+ public void recordWorkflowComplete(WorkflowModel workflow) {
+ String workflowName = workflow.getWorkflowName();
+ if (skipLabels) {
+ workflowName = "None";
+ }
+ long duration = workflow.getEndTime() - workflow.getCreateTime();
+ getTimer(
+ "workflow_completed",
+ "workflowName",
+ workflowName,
+ "status",
+ workflow.getStatus().toString())
+ .record(duration, TimeUnit.MILLISECONDS);
+ }
+
+ public void recordTaskComplete(TaskModel task) {
+ String taskType = task.getTaskDefName();
+ if (skipLabels) {
+ taskType = "None";
+ }
+ getTimer("task_completed", "taskType", taskType, "status", task.getStatus().toString())
+ .record((task.getEndTime() - task.getStartTime()), TimeUnit.MILLISECONDS);
+ }
+
+ private static List tags(String... additionalTags) {
+ List tags = new ArrayList<>();
+
+ for (int j = 0; j < additionalTags.length - 1; j++) {
+ String tk = additionalTags[j];
+ String tv = "" + additionalTags[j + 1];
+ if (!tv.isEmpty()) {
+ tags.add(Tag.of(tk, tv));
+ }
+ j++;
+ }
+ return tags;
+ }
+}
diff --git a/archive/src/main/resources/db/migration_archive_postgres/V99__initial_schema.sql b/archive/src/main/resources/db/migration_archive_postgres/V99__initial_schema.sql
new file mode 100644
index 0000000..64a5bb5
--- /dev/null
+++ b/archive/src/main/resources/db/migration_archive_postgres/V99__initial_schema.sql
@@ -0,0 +1,26 @@
+-- Workflow
+CREATE TABLE workflow_archive (
+ workflow_id varchar(255) NOT NULL,
+ created_on bigint,
+ modified_on bigint,
+ created_by varchar(255),
+ correlation_id varchar(255),
+ workflow_name varchar(255),
+ status varchar(255),
+ json_data TEXT,
+ index_data text[],
+ PRIMARY KEY (workflow_id)
+) with (autovacuum_vacuum_scale_factor = 0.0, autovacuum_vacuum_threshold = 10000);
+
+CREATE INDEX workflow_archive_workflow_name_index ON workflow_archive (workflow_name, status, correlation_id, created_on);
+CREATE INDEX workflow_archive_search_index ON workflow_archive USING gin(index_data);
+CREATE INDEX workflow_archive_created_on_index ON workflow_archive (created_on desc);
+
+-- Task Logs
+CREATE TABLE task_logs (
+ task_id varchar(255) NOT NULL,
+ seq bigserial,
+ log TEXT,
+ created_on bigint,
+ PRIMARY KEY (task_id, seq)
+);
\ No newline at end of file
diff --git a/archive/src/test/java/io/orkes/conductor/dao/archive/TestScheduler.java b/archive/src/test/java/io/orkes/conductor/dao/archive/TestScheduler.java
new file mode 100644
index 0000000..eabb005
--- /dev/null
+++ b/archive/src/test/java/io/orkes/conductor/dao/archive/TestScheduler.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.archive;
+
+import java.text.DecimalFormat;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.*;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+
+import com.netflix.conductor.common.metadata.workflow.StartWorkflowRequest;
+
+public class TestScheduler {
+
+ public static void main(String[] args) throws InterruptedException {
+ RestTemplate rt =
+ new RestTemplateBuilder()
+ .setReadTimeout(Duration.ofMillis(7000))
+ .setConnectTimeout(Duration.ofMillis(7000))
+ .build();
+
+ StartWorkflowRequest request = new StartWorkflowRequest();
+ request.setName("first_flow_all_simple");
+ DecimalFormat df = new DecimalFormat("000");
+ ExecutorService es = Executors.newFixedThreadPool(100);
+ int start = 1;
+ int maxSet = 150;
+ String host = "https://perf6.conductorworkflow.net/";
+ CountDownLatch latch = new CountDownLatch(maxSet);
+ HttpHeaders headers = new HttpHeaders();
+ headers.put(
+ "X-Authorization",
+ Collections.singletonList(
+ "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIiwiaXNzIjoiaHR0cHM6Ly9vcmtlcy10ZXN0LnVzLmF1dGgwLmNvbS8ifQ..Hhotod4LBfYqEZBc.AFl2Xa_AYk4NaWJzakRjuMx_wpmj-RSLaf7RzzFZxdo35jW3fXzA7RQ8pOevieLWZKgMRYPc6FWdyoFbIZXFUYZcQPUk83UzsigXQSB4g1PUUslmxv6betHQtBNt0PYaNLzwI5PF7QkzNWEdeIPa2b-IsTXniagk_fFeTRJmMdPmfqDDqGmU6kd1v3M12JOqVp5hNXGJirErz-9tB8uOySPXFVbiTCbz8mk_JA-B4LTUWzPzdyE6J7QqSHmsqjQZfkPNpCTYnEF958xLn1x3vZ2K9d84YYSYQTPU_ce3lZJeI3RbfoOp2fQL6KWzPIcPujRh.NdgKCYUDzhIZzfHinmMQdg"));
+ headers.setContentType(MediaType.APPLICATION_JSON);
+
+ for (int i = 0; i < 60; i++) {
+ int finalI = i;
+ es.submit(
+ () -> {
+ try {
+ for (int j = start; j <= maxSet; j++) {
+ Map sp = new HashMap<>();
+ sp.put(
+ "name",
+ String.format(
+ "test_second_%s_%s",
+ df.format(j), df.format(finalI)));
+ System.out.println(
+ String.format(
+ "test_second_%s_%s",
+ df.format(j), df.format(finalI)));
+
+ // rt.exchange("https://perf6.conductorworkflow.net/api/scheduler/schedules/" + sp.get("name"), HttpMethod.DELETE, new HttpEntity<>(headers), Void.class);
+
+ sp.put("cronExpression", String.format("%s * * ? * *", finalI));
+ sp.put("startWorkflowRequest", request);
+ HttpEntity> entity =
+ new HttpEntity<>(sp, headers);
+
+ ResponseEntity voidResponseEntity =
+ rt.exchange(
+ host + "api/scheduler/schedules",
+ HttpMethod.POST,
+ entity,
+ Void.class);
+ System.out.println(voidResponseEntity.getStatusCode());
+ }
+ } catch (RestClientException e) {
+ e.printStackTrace();
+ } finally {
+ latch.countDown();
+ System.out.println("latch: " + latch.getCount());
+ }
+ });
+ }
+ latch.await();
+ System.out.println("Latch: " + latch.getCount());
+ System.out.println("Done");
+ }
+
+ private static void createWorkflow(RestTemplate rt, HttpHeaders headers) {
+ Map sp = new HashMap<>();
+ sp.put("name", "first_flow");
+ HttpEntity> entity = new HttpEntity<>(sp, headers);
+ ResponseEntity exchange =
+ rt.exchange(
+ "https://perf5.orkesconductor.com/api/workflow",
+ HttpMethod.DELETE,
+ entity,
+ String.class);
+ System.out.println(exchange.getStatusCode());
+ }
+}
diff --git a/archive/src/test/java/io/orkes/conductor/dao/indexer/TestIndexValueExtractor.java b/archive/src/test/java/io/orkes/conductor/dao/indexer/TestIndexValueExtractor.java
new file mode 100644
index 0000000..ced1698
--- /dev/null
+++ b/archive/src/test/java/io/orkes/conductor/dao/indexer/TestIndexValueExtractor.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.indexer;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+import com.netflix.conductor.common.config.ObjectMapperProvider;
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+
+import static io.orkes.conductor.dao.indexer.IndexValueExtractor.getIndexWords;
+import static org.junit.jupiter.api.Assertions.*;
+
+public class TestIndexValueExtractor {
+
+ @Test
+ public void testIndexer() {
+ WorkflowModel model = new WorkflowModel();
+ model.getTasks().add(new TaskModel());
+
+ String uuid = UUID.randomUUID().toString();
+ model.getInput().put("correlation_id", uuid);
+ System.out.println("correlationId: " + uuid);
+ model.getInput().put("keep", "abcd");
+
+ String json =
+ "{\n"
+ + " \"response\": \"java.lang.Exception: I/O error on GET request for \\\"https://orkes-services.web.app2/data.json\\\": orkes-services.web.app2: nodename nor servname provided, or not known; nested exception is java.net.UnknownHostException: orkes-services.web.app2: nodename nor servname provided, or not known\"\n"
+ + "}";
+ try {
+ Map output = new ObjectMapperProvider().getObjectMapper().readValue(json, Map.class);
+ model.getOutput().putAll(output);
+ } catch (Exception e) {
+
+ }
+
+ for (int i = 0; i < 100; i++) {
+ model.getTasks().get(0).getOutputData().put("id" + i, UUID.randomUUID().toString());
+ }
+ Collection words = getIndexWords(model, 2, 50);
+
+ // Sine all the UUIDs are longer than max word length, they should all get filtered out
+ assertNotNull(words);
+ assertEquals(2, words.size());
+ assertTrue(words.contains("abcd"));
+ assertTrue(words.contains(uuid), uuid + " not in the list of words : " + words);
+
+ words = getIndexWords(model, 200, 50);
+ System.out.println(words);
+ System.out.println(words.size());
+ words.stream().forEach(System.out::println);
+
+ // All UUIDs shouldbe present
+ assertNotNull(words);
+ assertTrue(words.contains("https://orkes-services.web.app2/data.json"));
+ assertTrue(words.contains(uuid));
+ }
+}
diff --git a/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresArchiveDAOTest.java b/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresArchiveDAOTest.java
new file mode 100644
index 0000000..93f4b00
--- /dev/null
+++ b/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresArchiveDAOTest.java
@@ -0,0 +1,308 @@
+/*
+ * Copyright 2021 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres;
+
+import java.util.*;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import com.netflix.conductor.common.config.ObjectMapperProvider;
+import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+
+import io.orkes.conductor.dao.postgres.archive.PostgresArchiveDAO;
+import io.orkes.conductor.id.TimeBasedUUIDGenerator;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class PostgresArchiveDAOTest {
+
+ private final ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper();
+ private PostgresDAOTestUtil testPostgres;
+ private PostgresArchiveDAO archiveDAO;
+ public PostgreSQLContainer postgreSQLContainer;
+ public GenericContainer redis;
+ private TimeBasedUUIDGenerator timeBasedUUIDGenerator = new TimeBasedUUIDGenerator();
+
+ @BeforeAll
+ public void setup() {
+ redis =
+ new GenericContainer(DockerImageName.parse("redis:6.2.6-alpine"))
+ .withExposedPorts(6379);
+ redis.start();
+
+ ConductorProperties conductorProperties = new ConductorProperties();
+
+ postgreSQLContainer =
+ new PostgreSQLContainer<>(DockerImageName.parse("postgres:11.15-alpine"))
+ .withDatabaseName("conductor");
+ postgreSQLContainer.start();
+ testPostgres = new PostgresDAOTestUtil(postgreSQLContainer, objectMapper);
+
+ archiveDAO =
+ new PostgresArchiveDAO(
+ testPostgres.getObjectMapper(),
+ testPostgres.getDataSource(),
+ testPostgres.getDataSource());
+ }
+
+ @AfterAll
+ public void teardown() {
+ testPostgres.getDataSource().close();
+ postgreSQLContainer.stop();
+ }
+
+ @Test
+ public void testIndexLargeDoc() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("pending_count_correlation_jtest");
+
+ WorkflowModel workflow = createTestWorkflow();
+ Map output = workflow.getTasks().get(0).getOutputData();
+ for (int i = 0; i < 100; i++) {
+ output.put("key_" + i, generateRandomString());
+ }
+
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ }
+
+ private static String generateRandomString() {
+ Random random = new Random();
+ int wordCount = 5;
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < wordCount; i++) {
+ sb.append(generateRandomWord());
+ sb.append(" ");
+ }
+ return sb.toString();
+ }
+
+ private static String generateRandomWord() {
+ int leftLimit = 48; // numeral '0'
+ int rightLimit = 122; // letter 'z'
+ Random random = new Random();
+ int targetStringLength = 15;
+
+ String generatedString =
+ random.ints(leftLimit, rightLimit + 1)
+ .filter(i -> (i <= 57 || i >= 65) && (i <= 90 || i >= 97))
+ .limit(targetStringLength)
+ .collect(
+ StringBuilder::new,
+ StringBuilder::appendCodePoint,
+ StringBuilder::append)
+ .toString();
+
+ return generatedString;
+ }
+
+ @Test
+ public void testOlderDataIndex() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("pending_count_correlation_jtest");
+
+ WorkflowModel workflow = createTestWorkflow();
+ workflow.setWorkflowDefinition(def);
+ workflow.setWorkflowId(UUID.randomUUID().toString());
+
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ WorkflowModel found = archiveDAO.getWorkflow(workflow.getWorkflowId(), false);
+ assertNotNull(found);
+ assertEquals(workflow.getWorkflowId(), found.getWorkflowId());
+ }
+
+ @Test
+ public void testIndexWorkflow() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("pending_count_correlation_jtest");
+
+ WorkflowModel workflow = createTestWorkflow();
+ workflow.setWorkflowDefinition(def);
+
+ generateWorkflows(workflow, 10);
+ List bycorrelationId =
+ archiveDAO.getWorkflowIdsByCorrelationId(
+ "pending_count_correlation_jtest", "corr001", true, true);
+ assertNotNull(bycorrelationId);
+ assertEquals(10, bycorrelationId.size());
+ System.out.println("Workflow Ids: " + bycorrelationId);
+
+ List bycorrelationId2 =
+ archiveDAO.getWorkflowIdsByCorrelationId(
+ "pending_count_correlation_jtest", "corr001", true, true);
+ System.out.println("Workflow Ids: " + bycorrelationId2);
+ System.out.println("Workflow Ids: " + (bycorrelationId.size() == bycorrelationId2.size()));
+
+ workflow.setStatus(WorkflowModel.Status.COMPLETED);
+ workflow.setUpdatedTime(System.currentTimeMillis());
+ workflow.getTasks().forEach(t -> t.setStatus(TaskModel.Status.COMPLETED));
+ workflow.setUpdatedTime(System.currentTimeMillis());
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ WorkflowModel found = archiveDAO.getWorkflow(workflow.getWorkflowId(), false);
+ assertNotNull(found);
+ assertNotNull(workflow.getTasks());
+ assertFalse(workflow.getTasks().isEmpty());
+
+ // Updating it back to running status shouldn't do anything!
+ workflow.setStatus(WorkflowModel.Status.RUNNING);
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ found = archiveDAO.getWorkflow(workflow.getWorkflowId(), false);
+ assertNotNull(found);
+ assertEquals(WorkflowModel.Status.COMPLETED, found.getStatus());
+ }
+
+ @Test
+ public void testTablePartitioning() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("pending_count_correlation_jtest");
+
+ WorkflowModel workflow = createTestWorkflow();
+ workflow.setWorkflowDefinition(def);
+ workflow.setWorkflowId("4c66564d-c1e8-11ec-8f29-d2403f37b380");
+ workflow.setStatus(WorkflowModel.Status.COMPLETED);
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ WorkflowModel found = archiveDAO.getWorkflow(workflow.getWorkflowId(), false);
+ assertNotNull(found);
+ assertEquals(WorkflowModel.Status.COMPLETED, found.getStatus());
+ }
+
+ @Test
+ public void testFindByCorrelationId() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("pending_count_correlation_jtest");
+ String correlationId = "correlation_id#001";
+
+ WorkflowModel workflow1 = createTestWorkflow();
+ workflow1.setWorkflowDefinition(def);
+ workflow1.setWorkflowId(timeBasedUUIDGenerator.generate());
+ workflow1.setStatus(WorkflowModel.Status.RUNNING);
+ workflow1.setCorrelationId(correlationId);
+ archiveDAO.createOrUpdateWorkflow(workflow1);
+
+ WorkflowModel workflow2 = createTestWorkflow();
+ workflow2.setWorkflowDefinition(def);
+ workflow2.setWorkflowId(timeBasedUUIDGenerator.generate());
+ workflow2.setStatus(WorkflowModel.Status.COMPLETED);
+ workflow2.setCorrelationId(correlationId);
+ archiveDAO.createOrUpdateWorkflow(workflow2);
+
+ List found =
+ archiveDAO.getWorkflowIdsByCorrelationId(def.getName(), correlationId, true, true);
+ assertNotNull(found);
+ assertEquals(2, found.size());
+ assertTrue(found.contains(workflow1.getWorkflowId()));
+ assertTrue(found.contains(workflow2.getWorkflowId()));
+
+ found = archiveDAO.getWorkflowIdsByCorrelationId(def.getName(), correlationId, false, true);
+ assertNotNull(found);
+ assertEquals(1, found.size());
+ assertTrue(found.contains(workflow1.getWorkflowId()));
+ }
+
+ protected List generateWorkflows(WorkflowModel workflow, int count) {
+ List workflowIds = new ArrayList<>();
+ for (int i = 0; i < count; i++) {
+ String workflowId = new TimeBasedUUIDGenerator().generate();
+ workflow.setWorkflowId(workflowId);
+ workflow.setCorrelationId("corr001");
+ workflow.setStatus(WorkflowModel.Status.RUNNING);
+ workflow.setCreateTime(System.currentTimeMillis());
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ workflowIds.add(workflowId);
+ }
+ return workflowIds;
+ }
+
+ protected WorkflowModel createTestWorkflow() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("Junit Workflow");
+ def.setVersion(3);
+ def.setSchemaVersion(2);
+
+ WorkflowModel workflow = new WorkflowModel();
+ workflow.setWorkflowDefinition(def);
+ workflow.setCorrelationId("correlationX");
+ workflow.setCreatedBy("junit_tester");
+ workflow.setEndTime(200L);
+
+ Map input = new HashMap<>();
+ input.put("param1", "param1 value");
+ input.put("param2", 100);
+ workflow.setInput(input);
+
+ Map output = new HashMap<>();
+ output.put("ouput1", "output 1 value");
+ output.put("op2", 300);
+ workflow.setOutput(output);
+
+ workflow.setOwnerApp("workflow");
+ workflow.setParentWorkflowId("parentWorkflowId");
+ workflow.setParentWorkflowTaskId("parentWFTaskId");
+ workflow.setReasonForIncompletion("missing recipe");
+ workflow.setReRunFromWorkflowId("re-run from id1");
+ workflow.setCreateTime(90L);
+ workflow.setStatus(WorkflowModel.Status.FAILED);
+ workflow.setWorkflowId(timeBasedUUIDGenerator.generate());
+
+ List tasks = new LinkedList<>();
+
+ TaskModel task = new TaskModel();
+ task.setScheduledTime(1L);
+ task.setSeq(1);
+ task.setTaskId(timeBasedUUIDGenerator.generate());
+ task.setReferenceTaskName("t1");
+ task.setWorkflowInstanceId(workflow.getWorkflowId());
+ task.setTaskDefName("task1");
+ task.setStatus(TaskModel.Status.COMPLETED);
+
+ TaskModel task2 = new TaskModel();
+ task2.setScheduledTime(2L);
+ task2.setSeq(2);
+ task2.setTaskId(timeBasedUUIDGenerator.generate());
+ task2.setReferenceTaskName("t2");
+ task2.setWorkflowInstanceId(workflow.getWorkflowId());
+ task2.setTaskDefName("task2");
+ task2.setStatus(TaskModel.Status.COMPLETED);
+
+ TaskModel task3 = new TaskModel();
+ task3.setScheduledTime(2L);
+ task3.setSeq(3);
+ task3.setTaskId(timeBasedUUIDGenerator.generate());
+ task3.setReferenceTaskName("t3");
+ task3.setWorkflowInstanceId(workflow.getWorkflowId());
+ task3.setTaskDefName("task3");
+ task3.setStatus(TaskModel.Status.IN_PROGRESS);
+
+ tasks.add(task);
+ tasks.add(task2);
+ tasks.add(task3);
+
+ workflow.setTasks(tasks);
+
+ workflow.setUpdatedBy("junit_tester");
+ workflow.setUpdatedTime(800L);
+
+ return workflow;
+ }
+}
diff --git a/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresArchivePerformanceTest.java b/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresArchivePerformanceTest.java
new file mode 100644
index 0000000..8f89abc
--- /dev/null
+++ b/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresArchivePerformanceTest.java
@@ -0,0 +1,215 @@
+/*
+ * Copyright 2021 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres;
+
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.netflix.conductor.common.config.ObjectMapperProvider;
+import com.netflix.conductor.common.metadata.tasks.TaskDef;
+import com.netflix.conductor.common.metadata.tasks.TaskType;
+import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
+import com.netflix.conductor.common.metadata.workflow.WorkflowTask;
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+
+import io.orkes.conductor.dao.postgres.archive.PostgresArchiveDAO;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.javafaker.*;
+import com.zaxxer.hikari.HikariDataSource;
+
+public class PostgresArchivePerformanceTest {
+
+ private PostgresArchiveDAO archiveDAO;
+ private ObjectMapper objectMapper = new ObjectMapperProvider().getObjectMapper();
+ private HikariDataSource dataSource;
+ private static final Faker[] fakers =
+ new Faker[] {
+ new Faker(new Locale("en_US")),
+ new Faker(new Locale("nb-NO")),
+ new Faker(new Locale("zh-CN")),
+ new Faker(new Locale("fi-FI"))
+ };
+
+ private static final int TASK_COUNT = 5;
+
+ private Faker faker;
+
+ public PostgresArchivePerformanceTest() {
+ dataSource = new HikariDataSource();
+ dataSource.setJdbcUrl("jdbc:postgresql://localhost/neel");
+ dataSource.setUsername("postgres");
+ dataSource.setPassword("postgres");
+ dataSource.setAutoCommit(false);
+ dataSource.setMaximumPoolSize(10);
+ archiveDAO = new PostgresArchiveDAO(objectMapper, dataSource, dataSource);
+ faker = new Faker();
+ }
+
+ public void generateWorkflowsAndIndex(int count) {
+ WorkflowDef workflowDef = generateWorkflowDef();
+ for (int i = 0; i < count; i++) {
+ WorkflowModel workflow = createWorkflowRun(workflowDef);
+ archiveDAO.createOrUpdateWorkflow(workflow);
+ System.out.println(workflow.getStatus() + " : " + workflow.getWorkflowId());
+ }
+ }
+
+ private WorkflowDef generateWorkflowDef() {
+ WorkflowDef def = new WorkflowDef();
+ def.setName("load_test_workflow");
+ def.setVersion(1);
+ def.setOwnerEmail("loadtest@orkes.io");
+ def.setDescription("load testing workflow");
+
+ for (int i = 0; i < TASK_COUNT; i++) {
+ WorkflowTask workflowTask = new WorkflowTask();
+ workflowTask.setType(TaskType.SIMPLE.name());
+ workflowTask.setTaskReferenceName("task_" + i);
+ workflowTask.setName("task-" + i);
+ TaskDef taskDef = new TaskDef();
+ taskDef.setName("task-" + i);
+ taskDef.setOwnerEmail("dev@orkes.io");
+ workflowTask.setTaskDefinition(taskDef);
+ def.getTasks().add(workflowTask);
+ }
+ return def;
+ }
+
+ protected WorkflowModel createWorkflowRun(WorkflowDef workflowDef) {
+
+ WorkflowModel workflow = new WorkflowModel();
+ String workflowId = UUID.randomUUID().toString();
+ workflow.setWorkflowId(workflowId);
+ workflow.setWorkflowDefinition(workflowDef);
+ workflow.setCorrelationId(faker.gameOfThrones().character());
+ workflow.setCreatedBy("junit_tester");
+ workflow.setEndTime(200L);
+
+ workflow.setInput(getAddress(faker));
+ workflow.setOutput(getCommerceData(faker));
+
+ workflow.setOwnerApp("workflow");
+ workflow.setReRunFromWorkflowId(UUID.randomUUID().toString());
+ workflow.setCreateTime(System.currentTimeMillis() - 10000L);
+ workflow.setWorkflowId(UUID.randomUUID().toString());
+
+ List tasks = new LinkedList<>();
+ boolean failed = false;
+ String reason = null;
+ for (int i = 0; i < TASK_COUNT; i++) {
+ TaskModel task = new TaskModel();
+ task.setScheduledTime(1L);
+ task.setSeq(i + 1);
+
+ task.setTaskId(UUID.randomUUID().toString());
+ task.setReferenceTaskName("task_" + i);
+ task.setWorkflowInstanceId(workflow.getWorkflowId());
+ task.setTaskDefName("task-" + i);
+ task.setWorkerId(faker.idNumber().valid());
+ task.setTaskType(TaskType.SIMPLE.name());
+ task.setWorkflowTask(workflowDef.getTasks().get(i));
+ task.setWorkflowType(workflow.getWorkflowName());
+
+ int status = new Random().nextInt(10000);
+ if (status >= 20) {
+ task.setStatus(TaskModel.Status.COMPLETED);
+ } else if (status < 10) {
+ task.setStatus(TaskModel.Status.FAILED);
+ failed = true;
+ reason = faker.chuckNorris().fact();
+ task.setReasonForIncompletion(reason);
+ } else if (status >= 10 && status < 20) {
+ task.setStatus(TaskModel.Status.TIMED_OUT);
+ failed = true;
+ reason = faker.chuckNorris().fact();
+ task.setReasonForIncompletion(reason);
+ }
+
+ Faker faker2 = fakers[new Random().nextInt(fakers.length)];
+ task.getOutputData().putAll(getCommerceData(faker2));
+ task.getOutputData().putAll(getAddress(faker2));
+ task.getOutputData().putAll(getDemographics(faker2));
+ task.getInputData().putAll(getAddress(faker2));
+
+ tasks.add(task);
+ }
+ workflow.setTasks(tasks);
+
+ if (failed) {
+ workflow.setStatus(WorkflowModel.Status.FAILED);
+ workflow.setReasonForIncompletion(reason);
+ } else {
+ workflow.setStatus(WorkflowModel.Status.COMPLETED);
+ }
+
+ workflow.setUpdatedBy("junit_tester");
+ workflow.setUpdatedTime(System.currentTimeMillis());
+
+ return workflow;
+ }
+
+ private Map getCommerceData(Faker faker) {
+ Commerce commerce = faker.commerce();
+ Map data = new HashMap<>();
+ data.put("department", commerce.department());
+ data.put("price", commerce.price());
+ data.put("material", commerce.material());
+ data.put("promo_code", commerce.promotionCode());
+ data.put("product_name", commerce.productName());
+ return data;
+ }
+
+ private Map getAddress(Faker faker) {
+ Address address = faker.address();
+ Map data = new HashMap<>();
+
+ data.put("apt", address.buildingNumber());
+ data.put("full_address", address.fullAddress());
+ data.put("city", address.city());
+ data.put("street", address.streetAddress());
+ data.put("country", address.country());
+ data.put("zip", address.zipCode());
+
+ return data;
+ }
+
+ private Map getDemographics(Faker faker) {
+ Demographic demographic = faker.demographic();
+ Map data = new HashMap<>();
+
+ data.put("demonym", demographic.demonym());
+ data.put("education", demographic.educationalAttainment());
+ data.put("marital_status", demographic.maritalStatus());
+ data.put("race", demographic.race());
+ data.put("sex", demographic.sex());
+ return data;
+ }
+
+ public static void main(String[] args) {
+ System.out.println("Start");
+ PostgresArchivePerformanceTest tester = new PostgresArchivePerformanceTest();
+ ExecutorService es = Executors.newFixedThreadPool(100);
+ for (int i = 0; i < 1000; i++) {
+ es.submit(
+ () -> {
+ tester.generateWorkflowsAndIndex(1000);
+ });
+ }
+ es.shutdown();
+
+ System.out.println("Done");
+ }
+}
diff --git a/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresDAOTestUtil.java b/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresDAOTestUtil.java
new file mode 100644
index 0000000..6fa204d
--- /dev/null
+++ b/archive/src/test/java/io/orkes/conductor/dao/postgres/PostgresDAOTestUtil.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres;
+
+import java.nio.file.Paths;
+import java.time.Duration;
+
+import javax.sql.DataSource;
+
+import org.flywaydb.core.Flyway;
+import org.flywaydb.core.api.configuration.FluentConfiguration;
+import org.testcontainers.containers.PostgreSQLContainer;
+
+import com.netflix.conductor.postgres.config.PostgresProperties;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zaxxer.hikari.HikariDataSource;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class PostgresDAOTestUtil {
+
+ private final HikariDataSource dataSource;
+ private final PostgresProperties properties = mock(PostgresProperties.class);
+ private final ObjectMapper objectMapper;
+
+ public PostgresDAOTestUtil(
+ PostgreSQLContainer postgreSQLContainer, ObjectMapper objectMapper) {
+
+ this.objectMapper = objectMapper;
+
+ this.dataSource = new HikariDataSource();
+ dataSource.setJdbcUrl(postgreSQLContainer.getJdbcUrl());
+ dataSource.setUsername(postgreSQLContainer.getUsername());
+ dataSource.setPassword(postgreSQLContainer.getPassword());
+ dataSource.setAutoCommit(false);
+ // Prevent DB from getting exhausted during rapid testing
+ dataSource.setMaximumPoolSize(8);
+
+ when(properties.getTaskDefCacheRefreshInterval()).thenReturn(Duration.ofSeconds(60));
+
+ flywayMigrate(dataSource);
+ }
+
+ private void flywayMigrate(DataSource dataSource) {
+ FluentConfiguration fluentConfiguration =
+ Flyway.configure()
+ .table("schema_version")
+ .locations(Paths.get("db", "migration_archive_postgres").toString())
+ .dataSource(dataSource)
+ .mixed(true)
+ .placeholderReplacement(false);
+
+ Flyway flyway = fluentConfiguration.load();
+ flyway.migrate();
+ }
+
+ public HikariDataSource getDataSource() {
+ return dataSource;
+ }
+
+ public PostgresProperties getTestProperties() {
+ return properties;
+ }
+
+ public ObjectMapper getObjectMapper() {
+ return objectMapper;
+ }
+}
diff --git a/archive/src/test/java/io/orkes/conductor/dao/postgres/UuidUtil.java b/archive/src/test/java/io/orkes/conductor/dao/postgres/UuidUtil.java
new file mode 100644
index 0000000..ed18fd9
--- /dev/null
+++ b/archive/src/test/java/io/orkes/conductor/dao/postgres/UuidUtil.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package io.orkes.conductor.dao.postgres;
+
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.util.NetUtils;
+import org.apache.logging.log4j.core.util.Patterns;
+import org.apache.logging.log4j.status.StatusLogger;
+import org.apache.logging.log4j.util.PropertiesUtil;
+
+/**
+ * Generates a unique ID. The generated UUID will be unique for approximately 8,925 years so long as
+ * less than 10,000 IDs are generated per millisecond on the same device (as identified by its MAC
+ * address).
+ */
+public final class UuidUtil {
+
+ private static final long[] EMPTY_LONG_ARRAY = {};
+
+ /** System property that may be used to seed the UUID generation with an integer value. */
+ public static final String UUID_SEQUENCE = "org.apache.logging.log4j.uuidSequence";
+
+ private static final Logger LOGGER = StatusLogger.getLogger();
+
+ private static final String ASSIGNED_SEQUENCES = "org.apache.logging.log4j.assignedSequences";
+
+ private static final AtomicInteger COUNT = new AtomicInteger(0);
+ private static final long TYPE1 = 0x1000L;
+ private static final byte VARIANT = (byte) 0x80;
+ private static final int SEQUENCE_MASK = 0x3FFF;
+ private static final long NUM_100NS_INTERVALS_SINCE_UUID_EPOCH = 0x01b21dd213814000L;
+ private static final long INITIAL_UUID_SEQNO =
+ PropertiesUtil.getProperties().getLongProperty(UUID_SEQUENCE, 0);
+
+ private static final long LOW_MASK = 0xffffffffL;
+ private static final long MID_MASK = 0xffff00000000L;
+ private static final long HIGH_MASK = 0xfff000000000000L;
+ private static final int NODE_SIZE = 8;
+ private static final int SHIFT_2 = 16;
+ private static final int SHIFT_4 = 32;
+ private static final int SHIFT_6 = 48;
+ private static final int HUNDRED_NANOS_PER_MILLI = 10000;
+
+ private static final long LEAST = initialize(NetUtils.getMacAddress());
+
+ /* This class cannot be instantiated */
+ private UuidUtil() {}
+
+ /**
+ * Initializes this class
+ *
+ * @param mac MAC address
+ * @return Least
+ */
+ static long initialize(byte[] mac) {
+ final Random randomGenerator = new SecureRandom();
+ if (mac == null || mac.length == 0) {
+ mac = new byte[6];
+ randomGenerator.nextBytes(mac);
+ }
+ final int length = mac.length >= 6 ? 6 : mac.length;
+ final int index = mac.length >= 6 ? mac.length - 6 : 0;
+ final byte[] node = new byte[NODE_SIZE];
+ node[0] = VARIANT;
+ node[1] = 0;
+ for (int i = 2; i < NODE_SIZE; ++i) {
+ node[i] = 0;
+ }
+ System.arraycopy(mac, index, node, 2, length);
+ final ByteBuffer buf = ByteBuffer.wrap(node);
+ long rand = INITIAL_UUID_SEQNO;
+ String assigned = PropertiesUtil.getProperties().getStringProperty(ASSIGNED_SEQUENCES);
+ long[] sequences;
+ if (assigned == null) {
+ sequences = EMPTY_LONG_ARRAY;
+ } else {
+ final String[] array = assigned.split(Patterns.COMMA_SEPARATOR);
+ sequences = new long[array.length];
+ int i = 0;
+ for (final String value : array) {
+ sequences[i] = Long.parseLong(value);
+ ++i;
+ }
+ }
+ if (rand == 0) {
+ rand = randomGenerator.nextLong();
+ }
+ rand &= SEQUENCE_MASK;
+ boolean duplicate;
+ do {
+ duplicate = false;
+ for (final long sequence : sequences) {
+ if (sequence == rand) {
+ duplicate = true;
+ break;
+ }
+ }
+ if (duplicate) {
+ rand = (rand + 1) & SEQUENCE_MASK;
+ }
+ } while (duplicate);
+ assigned = assigned == null ? Long.toString(rand) : assigned + ',' + Long.toString(rand);
+ System.setProperty(ASSIGNED_SEQUENCES, assigned);
+
+ return buf.getLong() | rand << SHIFT_6;
+ }
+
+ /**
+ * Generates Type 1 UUID. The time contains the number of 100NS intervals that have occurred
+ * since 00:00:00.00 UTC, 10 October 1582. Each UUID on a particular machine is unique to the
+ * 100NS interval until they rollover around 3400 A.D.
+ *
+ *
+ * - Digits 1-12 are the lower 48 bits of the number of 100 ns increments since the start of
+ * the UUID epoch.
+ *
- Digit 13 is the version (with a value of 1).
+ *
- Digits 14-16 are a sequence number that is incremented each time a UUID is generated.
+ *
- Digit 17 is the variant (with a value of binary 10) and 10 bits of the sequence number
+ *
- Digit 18 is final 16 bits of the sequence number.
+ *
- Digits 19-32 represent the system the application is running on.
+ *
+ *
+ * @return universally unique identifiers (UUID)
+ */
+ public static UUID getTimeBasedUuid(long offset) {
+
+ final long time =
+ (((System.currentTimeMillis() + offset) * HUNDRED_NANOS_PER_MILLI)
+ + NUM_100NS_INTERVALS_SINCE_UUID_EPOCH)
+ + (COUNT.incrementAndGet() % HUNDRED_NANOS_PER_MILLI);
+ final long timeLow = (time & LOW_MASK) << SHIFT_4;
+ final long timeMid = (time & MID_MASK) >> SHIFT_2;
+ final long timeHi = (time & HIGH_MASK) >> SHIFT_6;
+ final long most = timeLow | timeMid | TYPE1 | timeHi;
+ return new UUID(most, LEAST);
+ }
+}
diff --git a/archive/src/test/resources/application-testazurearchive.properties b/archive/src/test/resources/application-testazurearchive.properties
new file mode 100644
index 0000000..07785cb
--- /dev/null
+++ b/archive/src/test/resources/application-testazurearchive.properties
@@ -0,0 +1,15 @@
+conductor.db.type=redis_standalone
+conductor.redis.hosts=localhost:6379:us-east-1c
+
+conductor.archive.db.enabled=true
+conductor.archive.db.type=postgres
+conductor.archive.db.indexer.threadCount=1
+conductor.archive.db.indexer.pollingInterval=10
+spring.datasource.url=jdbc:tc:postgresql:11.15-alpine:///conductordb
+
+conductor.archive.db.document.store.type=azureblob
+conductor.archive.db.document.store.type.azureblob.connectionString=DefaultEndpointsProtocol=https;AccountName=aiademo;AccountKey=nMNQta5j9ILcRoALQhlFELQLJ0ILbXVu5NeEfkXWQT15WcupsNuQ1wrMZP1O5K5gWyyd3tJpQXpVln/zctt19A==;EndpointSuffix=core.windows.net
+
+conductor.grpc-server.port=8091
+es.set.netty.runtime.available.processors=false
+
diff --git a/archive/src/test/resources/application-tests3archive.properties b/archive/src/test/resources/application-tests3archive.properties
new file mode 100644
index 0000000..d75e8fc
--- /dev/null
+++ b/archive/src/test/resources/application-tests3archive.properties
@@ -0,0 +1,14 @@
+conductor.db.type=memory
+conductor.redis.hosts=host1:port:rack;host2:port:rack:host3:port:rack
+
+conductor.archive.db.enabled=true
+conductor.archive.db.type=postgres
+conductor.archive.db.indexer.threadCount=1
+conductor.archive.db.indexer.pollingInterval=10
+spring.datasource.url=jdbc:tc:postgresql:11.15-alpine:///conductordb
+
+conductor.archive.db.document.store.type=s3
+
+conductor.grpc-server.port=8091
+es.set.netty.runtime.available.processors=false
+
diff --git a/archive/src/test/resources/drop_all.sql b/archive/src/test/resources/drop_all.sql
new file mode 100644
index 0000000..a4f96d0
--- /dev/null
+++ b/archive/src/test/resources/drop_all.sql
@@ -0,0 +1 @@
+DROP SCHEMA public CASCADE;
\ No newline at end of file
diff --git a/archive/src/test/resources/logback.xml b/archive/src/test/resources/logback.xml
new file mode 100644
index 0000000..d1353c6
--- /dev/null
+++ b/archive/src/test/resources/logback.xml
@@ -0,0 +1,31 @@
+
+
+
+
+
+
+ %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
new file mode 100644
index 0000000..7cfa6cb
--- /dev/null
+++ b/build.gradle
@@ -0,0 +1,179 @@
+buildscript {
+ dependencies {
+ classpath "org.springframework.boot:spring-boot-gradle-plugin:2.5.7"
+ classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.+'
+ }
+}
+
+plugins {
+ id 'io.spring.dependency-management' version '1.0.11.RELEASE'
+}
+
+ext {
+ group = 'io.orkes.queue'
+ appVersion = '0.0.1-SNAPSHOT'
+ springBootVersion = '2.5.6'
+
+ versions = [
+ conductorfork : '3.10.7',
+ revTestContainer : '1.17.2',
+ revGuava : '30.0-jre',
+ log4j : '2.17.1',
+ revJedis : '3.3.0',
+ revMockServerClient : '5.12.0',
+ revCommonsLang : '3.12.0',
+ revLombok : '1.18.24',
+ revLucene : '7.7.3',
+ revSpectator : '0.122.0',
+ revOpenapi : '1.6.11',
+ revAwsSdk : '1.12.153',
+ revProtoBuf : '3.13.0',
+ revRarefiedRedis : '0.0.17'
+ ]
+}
+
+def relVersion = System.getenv('REL_VER')
+if (relVersion) {
+ println "Inferred version from env variable 'REL_VER': $relVersion"
+ appVersion = relVersion
+}
+
+subprojects {
+
+ group = 'io.orkes.queue'
+ version = "${appVersion}"
+
+ apply plugin: 'java'
+ apply plugin: 'io.spring.dependency-management'
+ apply plugin: 'maven-publish'
+ apply plugin: 'signing'
+ apply plugin: 'com.diffplug.spotless'
+
+ repositories {
+ mavenCentral()
+ }
+
+ java {
+ withSourcesJar()
+ withJavadocJar()
+ }
+
+ configurations {
+ compileOnly {
+ extendsFrom annotationProcessor
+ }
+ testCompileOnly {
+ extendsFrom annotationProcessor
+ }
+ all {
+ exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl'
+ exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+ }
+ }
+
+ dependencies {
+ implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}!!"
+ implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}!!"
+ implementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}!!"
+ implementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}!!"
+ implementation "org.apache.logging.log4j:log4j-web:${versions.log4j}!!"
+ implementation "org.apache.logging.log4j:log4j-to-slf4j:${versions.log4j}!!"
+
+ compileOnly "org.projectlombok:lombok:${versions.revLombok}"
+ annotationProcessor "org.projectlombok:lombok:${versions.revLombok}"
+ testAnnotationProcessor "org.projectlombok:lombok:${versions.revLombok}"
+ implementation "org.apache.commons:commons-lang3:${versions.revCommonsLang}"
+ }
+
+ dependencyManagement {
+ imports {
+ mavenBom(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES)
+ }
+ }
+
+ test {
+ useJUnitPlatform()
+ testLogging {
+ events = ["SKIPPED", "FAILED"]
+ exceptionFormat = "full"
+ showStandardStreams = false
+ }
+ }
+
+
+ compileJava {
+ sourceCompatibility = 11
+ targetCompatibility = 11
+ }
+
+ spotless {
+ java {
+ googleJavaFormat().aosp()
+ removeUnusedImports()
+ importOrder('java', 'javax', 'org', 'com.netflix', 'io.orkes','', '\\#com.netflix', '\\#')
+ licenseHeaderFile("$rootDir/licenseheader.txt")
+ }
+ }
+ build.dependsOn(spotlessApply)
+
+
+ publishing {
+ publications {
+ mavenJava(MavenPublication) {
+ from components.java
+ pom {
+ name = 'Orkes Conductor Community'
+ description = 'Orkes supported version of open source Netflix Conductor'
+ url = 'https://github.com/orkes-io/orkes-conductor-community'
+ scm {
+ connection = 'scm:git://github.com/orkes-io/orkes-conductor-community.git'
+ developerConnection = 'scm:git://github.com/orkes-io/orkes-conductor-community.git'
+ url = 'https://github.com/orkes-io/orkes-conductor-community'
+ }
+ licenses {
+ license {
+ name = 'Orkes Community License'
+ url = 'https://github.com/orkes-io/licenses/community/LICENSE.txt'
+ }
+ }
+ developers {
+ developer {
+ organization = 'Orkes'
+ organizationUrl = 'https://orkes.io'
+ name = 'Orkes Development Team'
+ email = 'developers@orkes.io'
+ }
+ }
+ }
+ }
+ }
+
+ repositories {
+ maven {
+ if (project.hasProperty("mavenCentral")) {
+ println "Publishing to Sonatype Repository"
+ url = "https://s01.oss.sonatype.org/${project.version.endsWith('-SNAPSHOT') ? "content/repositories/snapshots/" : "service/local/staging/deploy/maven2/"}"
+ credentials {
+ username project.properties.username
+ password project.properties.password
+ }
+ }
+ }
+ }
+
+ def signingKeyId = findProperty('signingKeyId')
+ if(signingKeyId) {
+ println 'Signing the artifact with keys'
+ signing {
+ def signingKey = findProperty('signingKey')
+ def signingPassword = findProperty('signingPassword')
+ if (signingKeyId && signingKey && signingPassword) {
+ useInMemoryPgpKeys(signingKeyId, signingKey, signingPassword)
+ }
+
+ sign publishing.publications
+ }
+ }
+
+ }
+}
diff --git a/docker/DockerfileServer b/docker/DockerfileServer
new file mode 100644
index 0000000..d1548b9
--- /dev/null
+++ b/docker/DockerfileServer
@@ -0,0 +1,39 @@
+FROM alpine:3.16.2
+
+MAINTAINER Orkes Inc
+
+RUN apk add nginx
+
+RUN apk add openjdk11
+
+RUN apk add coreutils
+RUN apk add curl
+
+# Make app folders
+RUN mkdir -p /app/config /app/logs /app/libs /app/info
+
+# Add UI
+WORKDIR /usr/share/nginx/html
+RUN rm -rf ./*
+COPY tmp/ui/conductor/ui/build .
+COPY docker/config/nginx.conf /etc/nginx/http.d/default.conf
+
+# Startup script(s)
+COPY docker/config/startup.sh /app/startup.sh
+COPY docker/config/config.properties /app/config/config.properties
+COPY server/src/main/resources/banner.txt /app/config/banner.txt
+
+# JAR files
+COPY server/build/libs/orkes-conductor-server-boot.jar /app/libs/server.jar
+
+# Server version
+#COPY assembled/libs/server-version.txt* /app/info
+
+RUN chmod +x /app/startup.sh
+
+HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1
+
+EXPOSE 5000 8080
+
+CMD ["/app/startup.sh"]
+ENTRYPOINT ["/bin/sh"]
diff --git a/docker/DockerfileStandalone b/docker/DockerfileStandalone
new file mode 100644
index 0000000..8cc2aa2
--- /dev/null
+++ b/docker/DockerfileStandalone
@@ -0,0 +1,44 @@
+FROM alpine:3.16.2
+MAINTAINER Orkes Inc
+
+# Install software required to run conductor stack
+RUN apk add nginx
+RUN apk add coreutils
+RUN apk add openjdk11
+RUN apk add redis
+RUN apk add coreutils
+RUN apk add postgresql14
+
+# Make app folders
+RUN mkdir -p /app/config /app/logs /app/libs /app/info
+
+# Add UI
+WORKDIR /usr/share/nginx/html
+RUN rm -rf ./*
+COPY tmp/ui/conductor/ui/build .
+COPY docker/config/nginx.conf /etc/nginx/http.d/default.conf
+
+
+# Startup script(s)
+COPY docker/config/startup.sh /app/startup.sh
+COPY docker/config/config.properties /app/config/config.properties
+COPY docker/config/redis.conf /app/config/redis.conf
+COPY docker/config/start_all.sh /app/start_all.sh
+COPY server/src/main/resources/banner.txt /app/config/banner.txt
+
+# JAR files
+COPY server/build/libs/orkes-conductor-server-boot.jar /app/libs/server.jar
+
+RUN chmod +x /app/startup.sh
+RUN touch /app/logs/server.log
+
+# setup postgres
+RUN mkdir /run/postgresql
+RUN chown postgres:postgres /run/postgresql/
+
+HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1
+EXPOSE 5000 8080
+
+USER root
+CMD ["/app/start_all.sh"]
+ENTRYPOINT ["/bin/sh"]
diff --git a/docker/build-ui.sh b/docker/build-ui.sh
new file mode 100755
index 0000000..920ae43
--- /dev/null
+++ b/docker/build-ui.sh
@@ -0,0 +1,8 @@
+# clone and build conductor UI
+mkdir -p tmp/ui
+cd tmp/ui
+pwd
+git clone https://github.com/Netflix/conductor
+cd conductor/ui
+yarn install
+yarn build
\ No newline at end of file
diff --git a/docker/config/config.properties b/docker/config/config.properties
new file mode 100644
index 0000000..e69de29
diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf
new file mode 100644
index 0000000..652721d
--- /dev/null
+++ b/docker/config/nginx.conf
@@ -0,0 +1,44 @@
+server {
+ listen 5000;
+ server_name conductor;
+ server_tokens off;
+
+ gzip on;
+ gzip_vary on;
+ gzip_comp_level 6;
+ gzip_types text/plain text/css application/json application/x-javascript application/javascript text/xml application/xml application/rss+xml text/javascript image/svg+xml application/vnd.ms-fontobject application/x-font-ttf font/opentype;
+
+ location / {
+ add_header Referrer-Policy "strict-origin";
+ add_header X-Frame-Options "SAMEORIGIN";
+ add_header X-Content-Type-Options "nosniff";
+ add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;";
+ add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)";
+ # This would be the directory where your React app's static files are stored at
+ root /usr/share/nginx/html;
+ try_files $uri /index.html;
+ }
+
+ location /api {
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-NginX-Proxy true;
+ proxy_pass http://localhost:8080/api;
+ proxy_ssl_session_reuse off;
+ proxy_set_header Host $http_host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_redirect off;
+ }
+
+ location /swagger-ui {
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-NginX-Proxy true;
+ proxy_pass http://localhost:8080/swagger-ui;
+ proxy_ssl_session_reuse off;
+ proxy_set_header Host $http_host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_redirect off;
+ }
+
+}
\ No newline at end of file
diff --git a/docker/config/redis.conf b/docker/config/redis.conf
new file mode 100644
index 0000000..f43add6
--- /dev/null
+++ b/docker/config/redis.conf
@@ -0,0 +1 @@
+appendonly yes
\ No newline at end of file
diff --git a/docker/config/start_all.sh b/docker/config/start_all.sh
new file mode 100644
index 0000000..d6365b5
--- /dev/null
+++ b/docker/config/start_all.sh
@@ -0,0 +1,15 @@
+# configure and start redis
+mkdir -p /redis
+cd /redis
+nohup redis-server /app/config/redis.conf &
+
+# configure and start postgres
+mkdir -p /pgdata
+chown -R postgres:postgres /pgdata
+chmod 0700 /pgdata
+su postgres -c 'initdb -D /pgdata'
+su postgres -c 'pg_ctl start -D /pgdata'
+
+cat /app/config/banner.txt
+nohup /app/startup.sh &
+tail -f /app/logs/server.log
\ No newline at end of file
diff --git a/docker/config/startup.sh b/docker/config/startup.sh
new file mode 100644
index 0000000..5bd33b6
--- /dev/null
+++ b/docker/config/startup.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+echo "Starting Conductor Server and UI"
+echo "Running Nginx in background"
+# Start nginx as daemon
+nginx
+
+# Start the server
+cd /app/libs
+echo "Using config properties";
+export config_file=/app/config/config.properties
+
+if [[ -z "${JVM_MEMORY_SETTINGS}" ]]; then
+ JVM_MEMORY="-Xms512M -Xmx750M"
+else
+ JVM_MEMORY="${JVM_MEMORY_SETTINGS}"
+fi
+
+echo "Starting Conductor with $JVM_MEMORY memory settings"
+export LOG_FILE=/app/logs/server.log
+
+java $JVM_MEMORY -jar -DCONDUCTOR_CONFIG_FILE=$config_file server.jar
\ No newline at end of file
diff --git a/gradle.properties b/gradle.properties
new file mode 100644
index 0000000..af82e00
--- /dev/null
+++ b/gradle.properties
@@ -0,0 +1 @@
+org.gradle.parallel=true
\ No newline at end of file
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 0000000..7454180
Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 0000000..69a9715
--- /dev/null
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
new file mode 100755
index 0000000..1b6c787
--- /dev/null
+++ b/gradlew
@@ -0,0 +1,234 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+# * compound commands having a testable exit status, especially «case»;
+# * various built-in commands including «command», «set», and «ulimit».
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
+done
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+
+APP_NAME="Gradle"
+APP_BASE_NAME=${0##*/}
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+ echo "$*"
+} >&2
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD=$JAVA_HOME/jre/sh/java
+ else
+ JAVACMD=$JAVA_HOME/bin/java
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD=java
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
+ fi
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
+ done
+fi
+
+# Collect all arguments for the java command;
+# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# shell script including quotes and variable substitutions, so put them in
+# double quotes to make sure that they get re-expanded; and
+# * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
+
+exec "$JAVACMD" "$@"
diff --git a/gradlew.bat b/gradlew.bat
new file mode 100644
index 0000000..107acd3
--- /dev/null
+++ b/gradlew.bat
@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/licenseheader.txt b/licenseheader.txt
new file mode 100644
index 0000000..60e5820
--- /dev/null
+++ b/licenseheader.txt
@@ -0,0 +1,12 @@
+/*
+ * Copyright $YEAR Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
\ No newline at end of file
diff --git a/persistence/build.gradle b/persistence/build.gradle
new file mode 100644
index 0000000..5ec704b
--- /dev/null
+++ b/persistence/build.gradle
@@ -0,0 +1,31 @@
+dependencies {
+
+
+
+ implementation "com.netflix.conductor:conductor-common:${versions.conductorfork}"
+ implementation "com.netflix.conductor:conductor-core:${versions.conductorfork}"
+ compileOnly 'org.springframework.boot:spring-boot-starter'
+
+ implementation "redis.clients:jedis:${versions.revJedis}"
+
+ implementation "com.google.guava:guava:${versions.revGuava}"
+
+ //Object Mapper
+ implementation "com.fasterxml.jackson.core:jackson-databind"
+ implementation "com.fasterxml.jackson.core:jackson-core"
+
+ //In Memory redis
+ implementation "org.rarefiedredis.redis:redis-java:${versions.revRarefiedRedis}"
+
+ //In memory
+ implementation "com.jayway.jsonpath:json-path:2.4.0"
+
+
+ //Micrometer
+ implementation "io.micrometer:micrometer-core:1.7.5"
+
+ //spring
+ testImplementation 'org.springframework.boot:spring-boot-starter-test'
+ testImplementation 'org.junit.vintage:junit-vintage-engine'
+
+}
\ No newline at end of file
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/config/AnyRedisCondition.java b/persistence/src/main/java/com/netflix/conductor/redis/config/AnyRedisCondition.java
new file mode 100644
index 0000000..0789d72
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/config/AnyRedisCondition.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.config;
+
+import org.springframework.boot.autoconfigure.condition.AnyNestedCondition;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+
+public class AnyRedisCondition extends AnyNestedCondition {
+
+ public AnyRedisCondition() {
+ super(ConfigurationPhase.PARSE_CONFIGURATION);
+ }
+
+ @ConditionalOnProperty(name = "conductor.db.type", havingValue = "dynomite")
+ static class DynomiteClusterCondition {}
+
+ @ConditionalOnProperty(name = "conductor.db.type", havingValue = "memory")
+ static class InMemoryRedisCondition {}
+
+ @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_cluster")
+ static class RedisClusterConfiguration {}
+
+ @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_sentinel")
+ static class RedisSentinelConfiguration {}
+
+ @ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_standalone")
+ static class RedisStandaloneConfiguration {}
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/config/InMemoryRedisConfiguration.java b/persistence/src/main/java/com/netflix/conductor/redis/config/InMemoryRedisConfiguration.java
new file mode 100644
index 0000000..8913bc2
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/config/InMemoryRedisConfiguration.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.config;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+
+import com.netflix.conductor.redis.dynoqueue.LocalhostHostSupplier;
+import com.netflix.conductor.redis.jedis.JedisMock;
+import com.netflix.conductor.redis.jedis.JedisStandalone;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+import com.netflix.dyno.connectionpool.HostSupplier;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.commands.JedisCommands;
+
+@Configuration(proxyBeanMethods = false)
+@ConditionalOnProperty(name = "conductor.db.type", havingValue = "memory")
+public class InMemoryRedisConfiguration {
+
+ public static final JedisMock jedisMock = new JedisMock();
+
+ @Bean
+ public HostSupplier hostSupplier(RedisProperties properties) {
+ return new LocalhostHostSupplier(properties);
+ }
+
+ @Bean
+ public JedisMock jedisMock() {
+ return jedisMock;
+ }
+
+ @Bean
+ public JedisCommands jedisCommands() {
+ return new JedisStandalone(jedisPool());
+ }
+
+ @Bean
+ public JedisPool jedisPool() {
+ return new JedisPool() {
+ @Override
+ public Jedis getResource() {
+ return jedisMock;
+ }
+ };
+ }
+
+ @Primary
+ @Bean
+ public OrkesJedisProxy OrkesJedisProxy() {
+ System.out.println("OrkesJedisProxy created");
+ return new OrkesJedisProxy(jedisPool());
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/config/RedisClusterConfiguration.java b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisClusterConfiguration.java
new file mode 100644
index 0000000..1f5105b
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisClusterConfiguration.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.config;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier;
+import com.netflix.conductor.redis.jedis.JedisCluster;
+import com.netflix.dyno.connectionpool.Host;
+
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.Protocol;
+
+@Configuration(proxyBeanMethods = false)
+@ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_cluster")
+public class RedisClusterConfiguration {
+
+ private static final Logger log = LoggerFactory.getLogger(RedisClusterConfiguration.class);
+
+ // Same as redis.clients.jedis.BinaryJedisCluster
+ protected static final int DEFAULT_MAX_ATTEMPTS = 5;
+
+ @Bean
+ public JedisCluster getJedisCluster(RedisProperties properties) {
+ GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig<>();
+ genericObjectPoolConfig.setMaxTotal(properties.getMaxConnectionsPerHost());
+ ConfigurationHostSupplier hostSupplier = new ConfigurationHostSupplier(properties);
+ Set hosts =
+ hostSupplier.getHosts().stream()
+ .map(h -> new HostAndPort(h.getHostName(), h.getPort()))
+ .collect(Collectors.toSet());
+ String password = getPassword(hostSupplier.getHosts());
+
+ if (password != null) {
+ log.info("Connecting to Redis Cluster with AUTH");
+ }
+
+ return new JedisCluster(
+ new redis.clients.jedis.JedisCluster(
+ hosts,
+ Protocol.DEFAULT_TIMEOUT,
+ Protocol.DEFAULT_TIMEOUT,
+ DEFAULT_MAX_ATTEMPTS,
+ password,
+ null,
+ genericObjectPoolConfig,
+ properties.isSsl()));
+ }
+
+ private String getPassword(List hosts) {
+ return hosts.isEmpty() ? null : hosts.get(0).getPassword();
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/config/RedisProperties.java b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisProperties.java
new file mode 100644
index 0000000..2aa6776
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisProperties.java
@@ -0,0 +1,298 @@
+/*
+ * Copyright 2021 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.config;
+
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.convert.DurationUnit;
+import org.springframework.context.annotation.Configuration;
+
+import com.netflix.conductor.core.config.ConductorProperties;
+
+@ConfigurationProperties("conductor.redis")
+@Configuration
+public class RedisProperties {
+
+ private final ConductorProperties conductorProperties;
+
+ @Autowired
+ public RedisProperties(ConductorProperties conductorProperties) {
+ this.conductorProperties = conductorProperties;
+ }
+
+ /**
+ * Data center region. If hosting on Amazon the value is something like us-east-1, us-west-2
+ * etc.
+ */
+ private String dataCenterRegion = "us-east-1";
+
+ /**
+ * Local rack / availability zone. For AWS deployments, the value is something like us-east-1a,
+ * etc.
+ */
+ private String availabilityZone = "us-east-1c";
+
+ /** The name of the redis / dynomite cluster */
+ private String clusterName = "";
+
+ /** Dynomite Cluster details. Format is host:port:rack separated by semicolon */
+ private String hosts = null;
+
+ /** The prefix used to prepend workflow data in redis */
+ private String workflowNamespacePrefix = null;
+
+ /** The prefix used to prepend keys for queues in redis */
+ private String queueNamespacePrefix = null;
+
+ /**
+ * The domain name to be used in the key prefix for logical separation of workflow data and
+ * queues in a shared redis setup
+ */
+ private String keyspaceDomain = null;
+
+ /**
+ * The maximum number of connections that can be managed by the connection pool on a given
+ * instance
+ */
+ private int maxConnectionsPerHost = 10;
+
+ /** Database number. Defaults to a 0. Can be anywhere from 0 to 15 */
+ private int database = 0;
+
+ /**
+ * The maximum amount of time to wait for a connection to become available from the connection
+ * pool
+ */
+ private Duration maxTimeoutWhenExhausted = Duration.ofMillis(800);
+
+ /** The maximum retry attempts to use with this connection pool */
+ private int maxRetryAttempts = 0;
+
+ /** The read connection port to be used for connecting to dyno-queues */
+ private int queuesNonQuorumPort = 22122;
+
+ /** The time in seconds after which the in-memory task definitions cache will be refreshed */
+ @DurationUnit(ChronoUnit.SECONDS)
+ private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60);
+
+ /** The time to live in seconds for which the event execution will be persisted */
+ @DurationUnit(ChronoUnit.SECONDS)
+ private Duration eventExecutionPersistenceTTL = Duration.ofSeconds(60);
+
+ /** The time in seconds after which the in-memory metadata cache will be refreshed */
+ @DurationUnit(ChronoUnit.SECONDS)
+ private Duration metadataCacheRefreshInterval = Duration.ofSeconds(60);
+
+ // Maximum number of idle connections to be maintained
+ private int maxIdleConnections = 8;
+
+ // Minimum number of idle connections to be maintained
+ private int minIdleConnections = 5;
+
+ private long minEvictableIdleTimeMillis = 1800000;
+
+ private long timeBetweenEvictionRunsMillis = -1L;
+
+ private boolean testWhileIdle = false;
+
+ private int numTestsPerEvictionRun = 3;
+
+ private boolean ssl;
+
+ public int getNumTestsPerEvictionRun() {
+ return numTestsPerEvictionRun;
+ }
+
+ public void setNumTestsPerEvictionRun(int numTestsPerEvictionRun) {
+ this.numTestsPerEvictionRun = numTestsPerEvictionRun;
+ }
+
+ public boolean isTestWhileIdle() {
+ return testWhileIdle;
+ }
+
+ public void setTestWhileIdle(boolean testWhileIdle) {
+ this.testWhileIdle = testWhileIdle;
+ }
+
+ public long getMinEvictableIdleTimeMillis() {
+ return minEvictableIdleTimeMillis;
+ }
+
+ public void setMinEvictableIdleTimeMillis(long minEvictableIdleTimeMillis) {
+ this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis;
+ }
+
+ public long getTimeBetweenEvictionRunsMillis() {
+ return timeBetweenEvictionRunsMillis;
+ }
+
+ public void setTimeBetweenEvictionRunsMillis(long timeBetweenEvictionRunsMillis) {
+ this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis;
+ }
+
+ public int getMinIdleConnections() {
+ return minIdleConnections;
+ }
+
+ public void setMinIdleConnections(int minIdleConnections) {
+ this.minIdleConnections = minIdleConnections;
+ }
+
+ public int getMaxIdleConnections() {
+ return maxIdleConnections;
+ }
+
+ public void setMaxIdleConnections(int maxIdleConnections) {
+ this.maxIdleConnections = maxIdleConnections;
+ }
+
+ public String getDataCenterRegion() {
+ return dataCenterRegion;
+ }
+
+ public void setDataCenterRegion(String dataCenterRegion) {
+ this.dataCenterRegion = dataCenterRegion;
+ }
+
+ public String getAvailabilityZone() {
+ return availabilityZone;
+ }
+
+ public void setAvailabilityZone(String availabilityZone) {
+ this.availabilityZone = availabilityZone;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public void setClusterName(String clusterName) {
+ this.clusterName = clusterName;
+ }
+
+ public String getHosts() {
+ return hosts;
+ }
+
+ public void setHosts(String hosts) {
+ this.hosts = hosts;
+ }
+
+ public String getWorkflowNamespacePrefix() {
+ return workflowNamespacePrefix;
+ }
+
+ public void setWorkflowNamespacePrefix(String workflowNamespacePrefix) {
+ this.workflowNamespacePrefix = workflowNamespacePrefix;
+ }
+
+ public String getQueueNamespacePrefix() {
+ return queueNamespacePrefix;
+ }
+
+ public void setQueueNamespacePrefix(String queueNamespacePrefix) {
+ this.queueNamespacePrefix = queueNamespacePrefix;
+ }
+
+ public String getKeyspaceDomain() {
+ return keyspaceDomain;
+ }
+
+ public void setKeyspaceDomain(String keyspaceDomain) {
+ this.keyspaceDomain = keyspaceDomain;
+ }
+
+ public int getMaxConnectionsPerHost() {
+ return maxConnectionsPerHost;
+ }
+
+ public void setMaxConnectionsPerHost(int maxConnectionsPerHost) {
+ this.maxConnectionsPerHost = maxConnectionsPerHost;
+ }
+
+ public Duration getMaxTimeoutWhenExhausted() {
+ return maxTimeoutWhenExhausted;
+ }
+
+ public void setMaxTimeoutWhenExhausted(Duration maxTimeoutWhenExhausted) {
+ this.maxTimeoutWhenExhausted = maxTimeoutWhenExhausted;
+ }
+
+ public int getMaxRetryAttempts() {
+ return maxRetryAttempts;
+ }
+
+ public void setMaxRetryAttempts(int maxRetryAttempts) {
+ this.maxRetryAttempts = maxRetryAttempts;
+ }
+
+ public int getQueuesNonQuorumPort() {
+ return queuesNonQuorumPort;
+ }
+
+ public void setQueuesNonQuorumPort(int queuesNonQuorumPort) {
+ this.queuesNonQuorumPort = queuesNonQuorumPort;
+ }
+
+ public Duration getTaskDefCacheRefreshInterval() {
+ return taskDefCacheRefreshInterval;
+ }
+
+ public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) {
+ this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval;
+ }
+
+ public Duration getEventExecutionPersistenceTTL() {
+ return eventExecutionPersistenceTTL;
+ }
+
+ public void setEventExecutionPersistenceTTL(Duration eventExecutionPersistenceTTL) {
+ this.eventExecutionPersistenceTTL = eventExecutionPersistenceTTL;
+ }
+
+ public int getDatabase() {
+ return database;
+ }
+
+ public void setDatabase(int database) {
+ this.database = database;
+ }
+
+ public String getQueuePrefix() {
+ String prefix = getQueueNamespacePrefix() + "." + conductorProperties.getStack();
+ if (getKeyspaceDomain() != null) {
+ prefix = prefix + "." + getKeyspaceDomain();
+ }
+ return prefix;
+ }
+
+ public Duration getMetadataCacheRefreshInterval() {
+ return metadataCacheRefreshInterval;
+ }
+
+ public void setMetadataCacheRefreshInterval(Duration metadataCacheRefreshInterval) {
+ this.metadataCacheRefreshInterval = metadataCacheRefreshInterval;
+ }
+
+ public boolean isSsl() {
+ return ssl;
+ }
+
+ public void setSsl(boolean ssl) {
+ this.ssl = ssl;
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/config/RedisSentinelConfiguration.java b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisSentinelConfiguration.java
new file mode 100644
index 0000000..029ede8
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisSentinelConfiguration.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.config;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier;
+import com.netflix.conductor.redis.jedis.JedisSentinel;
+import com.netflix.dyno.connectionpool.Host;
+
+import redis.clients.jedis.JedisSentinelPool;
+import redis.clients.jedis.Protocol;
+
+@Configuration(proxyBeanMethods = false)
+@ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_sentinel")
+public class RedisSentinelConfiguration {
+
+ private static final Logger log = LoggerFactory.getLogger(RedisSentinelConfiguration.class);
+
+ @Bean
+ protected JedisSentinel getJedisSentinel(RedisProperties properties) {
+ GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig<>();
+ genericObjectPoolConfig.setMinIdle(properties.getMinIdleConnections());
+ genericObjectPoolConfig.setMaxIdle(properties.getMaxIdleConnections());
+ genericObjectPoolConfig.setMaxTotal(properties.getMaxConnectionsPerHost());
+ genericObjectPoolConfig.setTestWhileIdle(properties.isTestWhileIdle());
+ genericObjectPoolConfig.setMinEvictableIdleTimeMillis(
+ properties.getMinEvictableIdleTimeMillis());
+ genericObjectPoolConfig.setTimeBetweenEvictionRunsMillis(
+ properties.getTimeBetweenEvictionRunsMillis());
+ genericObjectPoolConfig.setNumTestsPerEvictionRun(properties.getNumTestsPerEvictionRun());
+ ConfigurationHostSupplier hostSupplier = new ConfigurationHostSupplier(properties);
+
+ log.info(
+ "Starting conductor server using redis_sentinel and cluster "
+ + properties.getClusterName());
+ Set sentinels = new HashSet<>();
+ for (Host host : hostSupplier.getHosts()) {
+ sentinels.add(host.getHostName() + ":" + host.getPort());
+ }
+ // We use the password of the first sentinel host as password and sentinelPassword
+ String password = getPassword(hostSupplier.getHosts());
+ if (password != null) {
+ return new JedisSentinel(
+ new JedisSentinelPool(
+ properties.getClusterName(),
+ sentinels,
+ genericObjectPoolConfig,
+ Protocol.DEFAULT_TIMEOUT,
+ Protocol.DEFAULT_TIMEOUT,
+ password,
+ properties.getDatabase(),
+ null,
+ Protocol.DEFAULT_TIMEOUT,
+ Protocol.DEFAULT_TIMEOUT,
+ password,
+ null));
+ } else {
+ return new JedisSentinel(
+ new JedisSentinelPool(
+ properties.getClusterName(),
+ sentinels,
+ genericObjectPoolConfig,
+ Protocol.DEFAULT_TIMEOUT,
+ null,
+ properties.getDatabase()));
+ }
+ }
+
+ private String getPassword(List hosts) {
+ return hosts.isEmpty() ? null : hosts.get(0).getPassword();
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/config/RedisStandaloneConfiguration.java b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisStandaloneConfiguration.java
new file mode 100644
index 0000000..5f61fdd
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/config/RedisStandaloneConfiguration.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.netflix.conductor.redis.dynoqueue.ConfigurationHostSupplier;
+import com.netflix.dyno.connectionpool.Host;
+
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
+import redis.clients.jedis.Protocol;
+
+@Configuration(proxyBeanMethods = false)
+@ConditionalOnProperty(name = "conductor.db.type", havingValue = "redis_standalone")
+public class RedisStandaloneConfiguration {
+
+ private static final Logger log = LoggerFactory.getLogger(RedisSentinelConfiguration.class);
+
+ @Bean
+ public JedisPool getJedisPool(RedisProperties redisProperties) {
+ JedisPoolConfig config = new JedisPoolConfig();
+ config.setMinIdle(2);
+ config.setMaxTotal(redisProperties.getMaxConnectionsPerHost());
+ log.info(
+ "Starting conductor server using redis_standalone - use SSL? {}",
+ redisProperties.isSsl());
+ ConfigurationHostSupplier hostSupplier = new ConfigurationHostSupplier(redisProperties);
+ Host host = hostSupplier.getHosts().get(0);
+
+ if (host.getPassword() != null) {
+ log.info("Connecting to Redis Standalone with AUTH");
+ return new JedisPool(
+ config,
+ host.getHostName(),
+ host.getPort(),
+ Protocol.DEFAULT_TIMEOUT,
+ host.getPassword(),
+ redisProperties.getDatabase(),
+ redisProperties.isSsl());
+ } else {
+ return new JedisPool(
+ config,
+ host.getHostName(),
+ host.getPort(),
+ Protocol.DEFAULT_TIMEOUT,
+ null,
+ redisProperties.getDatabase(),
+ redisProperties.isSsl());
+ }
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/BaseDynoDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/BaseDynoDAO.java
new file mode 100644
index 0000000..8655520
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/BaseDynoDAO.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.io.IOException;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.metrics.Monitors;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class BaseDynoDAO {
+
+ private static final String NAMESPACE_SEP = ".";
+ private static final String DAO_NAME = "redis";
+ private final String domain;
+ private final RedisProperties properties;
+ private final ConductorProperties conductorProperties;
+ protected OrkesJedisProxy orkesJedisProxy;
+ protected ObjectMapper objectMapper;
+
+ protected BaseDynoDAO(
+ OrkesJedisProxy jedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties) {
+ this.orkesJedisProxy = jedisProxy;
+ this.objectMapper = objectMapper;
+ this.conductorProperties = conductorProperties;
+ this.properties = properties;
+ this.domain = properties.getKeyspaceDomain();
+ }
+
+ String nsKey(String... nsValues) {
+ String rootNamespace = properties.getWorkflowNamespacePrefix();
+ StringBuilder namespacedKey = new StringBuilder();
+ if (StringUtils.isNotBlank(rootNamespace)) {
+ namespacedKey.append(rootNamespace).append(NAMESPACE_SEP);
+ }
+ String stack = conductorProperties.getStack();
+ if (StringUtils.isNotBlank(stack)) {
+ namespacedKey.append(stack).append(NAMESPACE_SEP);
+ }
+ if (StringUtils.isNotBlank(domain)) {
+ namespacedKey.append(domain).append(NAMESPACE_SEP);
+ }
+ for (String nsValue : nsValues) {
+ namespacedKey.append(nsValue).append(NAMESPACE_SEP);
+ }
+ return StringUtils.removeEnd(namespacedKey.toString(), NAMESPACE_SEP);
+ }
+
+ String toJson(Object value) {
+ try {
+ return objectMapper.writeValueAsString(value);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ T readValue(String json, Class clazz) {
+ try {
+ return objectMapper.readValue(json, clazz);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ void recordRedisDaoRequests(String action) {
+ recordRedisDaoRequests(action, "n/a", "n/a");
+ }
+
+ void recordRedisDaoRequests(String action, String taskType, String workflowType) {
+ Monitors.recordDaoRequests(DAO_NAME, action, taskType, workflowType);
+ }
+
+ void recordRedisDaoEventRequests(String action, String event) {
+ Monitors.recordDaoEventRequests(DAO_NAME, action, event);
+ }
+
+ void recordRedisDaoPayloadSize(String action, int size, String taskType, String workflowType) {
+ Monitors.recordDaoPayloadSize(
+ DAO_NAME,
+ action,
+ StringUtils.defaultIfBlank(taskType, ""),
+ StringUtils.defaultIfBlank(workflowType, ""),
+ size);
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/OrkesMetadataDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/OrkesMetadataDAO.java
new file mode 100644
index 0000000..13fc6aa
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/OrkesMetadataDAO.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.common.metadata.events.EventHandler;
+import com.netflix.conductor.common.metadata.tasks.TaskDef;
+import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.core.exception.ConflictException;
+import com.netflix.conductor.dao.EventHandlerDAO;
+import com.netflix.conductor.redis.config.AnyRedisCondition;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Component
+@Conditional(AnyRedisCondition.class)
+public class OrkesMetadataDAO extends RedisMetadataDAO {
+
+ private final ConcurrentMap missingTaskDefs = new ConcurrentHashMap<>();
+
+ private final long taskDefCacheTTL;
+ private final EventHandlerDAO eventHandlerDAO;
+ private static final String WORKFLOW_DEF = "WORKFLOW_DEF";
+ private static final String WORKFLOW_DEF_NAMES = "WORKFLOW_DEF_NAMES";
+ private static final String WAIT_FOR_EVENT = "WAIT_FOR_EVENT";
+
+ public OrkesMetadataDAO(
+ OrkesJedisProxy orkesJedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties,
+ EventHandlerDAO eventHandlerDAO) {
+ super(orkesJedisProxy, objectMapper, conductorProperties, properties);
+ taskDefCacheTTL = properties.getTaskDefCacheRefreshInterval().getSeconds() * 1000;
+ log.info("taskDefCacheTTL set to {}", taskDefCacheTTL);
+ this.eventHandlerDAO = eventHandlerDAO;
+ }
+
+ @Override
+ public void createWorkflowDef(WorkflowDef def) {
+ if (orkesJedisProxy.hexists(
+ nsKey(WORKFLOW_DEF, def.getName()), String.valueOf(def.getVersion()))) {
+ throw new ConflictException("Workflow with " + def.key() + " already exists!");
+ }
+ _createOrUpdate(def);
+ }
+
+ @Override
+ public void updateWorkflowDef(WorkflowDef def) {
+ _createOrUpdate(def);
+ }
+
+ private void _createOrUpdate(WorkflowDef workflowDef) {
+ // First set the workflow def
+ orkesJedisProxy.hset(
+ nsKey(WORKFLOW_DEF, workflowDef.getName()),
+ String.valueOf(workflowDef.getVersion()),
+ toJson(workflowDef));
+
+ orkesJedisProxy.sadd(nsKey(WORKFLOW_DEF_NAMES), workflowDef.getName());
+ for (var task : workflowDef.getTasks()) {
+ if (task.getType().equals(WAIT_FOR_EVENT)) {
+ EventHandler eventHandler = new EventHandler();
+ eventHandler.setEvent(task.getSink());
+ eventHandler.setName(workflowDef.getName() + "_" + task.getTaskReferenceName());
+ eventHandler.setActive(true);
+ EventHandler.Action action = new EventHandler.Action();
+ EventHandler.TaskDetails taskDetails = new EventHandler.TaskDetails();
+ taskDetails.setTaskRefName(task.getTaskReferenceName());
+ taskDetails.setWorkflowId("${targetWorkflowId}");
+ taskDetails.setOutput(Map.of("orkes_wait_for_event_task", true));
+ action.setComplete_task(taskDetails);
+ action.setAction(EventHandler.Action.Type.complete_task);
+ eventHandler.setActions(List.of(action));
+ // TODO: extend EventHandlerDAO with exists and updateOrInsert
+ if (eventHandlerDAO.getAllEventHandlers().stream()
+ .noneMatch(e -> e.getName().equals(eventHandler.getName()))) {
+ eventHandlerDAO.addEventHandler(eventHandler);
+ } else {
+ eventHandlerDAO.updateEventHandler(eventHandler);
+ }
+ }
+ }
+ recordRedisDaoRequests("storeWorkflowDef", "n/a", workflowDef.getName());
+ }
+
+ public TaskDef getTaskDef(String name, boolean ignoreCache) {
+ return ignoreCache ? super.getTaskDef(name) : getTaskDef(name);
+ }
+
+ @Override
+ public TaskDef getTaskDef(String name) {
+ Long lastChecked = missingTaskDefs.get(name);
+ // If the last check is NOT null, ie the task was reported missing earlier
+ // If so, check when was it last checked and if more than the configured TTL then refresh
+ long now = System.currentTimeMillis();
+ if (lastChecked != null && (now - lastChecked) < taskDefCacheTTL) {
+ return null;
+ }
+ TaskDef found = super.getTaskDef(name);
+ if (found == null) {
+ missingTaskDefs.put(name, now);
+ } else {
+ missingTaskDefs.remove(name);
+ }
+ return found;
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisEventHandlerDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisEventHandlerDAO.java
new file mode 100644
index 0000000..724d6d9
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisEventHandlerDAO.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.common.metadata.events.EventHandler;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.core.exception.ConflictException;
+import com.netflix.conductor.core.exception.NotFoundException;
+import com.netflix.conductor.dao.EventHandlerDAO;
+import com.netflix.conductor.redis.config.AnyRedisCondition;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+
+@Component
+@Conditional(AnyRedisCondition.class)
+public class RedisEventHandlerDAO extends BaseDynoDAO implements EventHandlerDAO {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RedisEventHandlerDAO.class);
+
+ private static final String EVENT_HANDLERS = "EVENT_HANDLERS";
+ private static final String EVENT_HANDLERS_BY_EVENT = "EVENT_HANDLERS_BY_EVENT";
+
+ public RedisEventHandlerDAO(
+ OrkesJedisProxy jedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties) {
+ super(jedisProxy, objectMapper, conductorProperties, properties);
+ }
+
+ @Override
+ public void addEventHandler(EventHandler eventHandler) {
+ Preconditions.checkNotNull(eventHandler.getName(), "Missing Name");
+ if (getEventHandler(eventHandler.getName()) != null) {
+ throw new ConflictException(
+ "EventHandler with name %s already exists!", eventHandler.getName());
+ }
+ index(eventHandler);
+ orkesJedisProxy.hset(nsKey(EVENT_HANDLERS), eventHandler.getName(), toJson(eventHandler));
+ recordRedisDaoRequests("addEventHandler");
+ }
+
+ @Override
+ public void updateEventHandler(EventHandler eventHandler) {
+ Preconditions.checkNotNull(eventHandler.getName(), "Missing Name");
+ EventHandler existing = getEventHandler(eventHandler.getName());
+ if (existing == null) {
+ throw new NotFoundException(
+ "EventHandler with name %s not found!", eventHandler.getName());
+ }
+ index(eventHandler);
+ orkesJedisProxy.hset(nsKey(EVENT_HANDLERS), eventHandler.getName(), toJson(eventHandler));
+ recordRedisDaoRequests("updateEventHandler");
+ }
+
+ @Override
+ public void removeEventHandler(String name) {
+ EventHandler existing = getEventHandler(name);
+ if (existing == null) {
+ throw new NotFoundException("EventHandler with name %s not found!", name);
+ }
+ orkesJedisProxy.hdel(nsKey(EVENT_HANDLERS), name);
+ recordRedisDaoRequests("removeEventHandler");
+ removeIndex(existing);
+ }
+
+ @Override
+ public List getAllEventHandlers() {
+ Map all = orkesJedisProxy.hgetAll(nsKey(EVENT_HANDLERS));
+ List handlers = new LinkedList<>();
+ all.forEach(
+ (key, json) -> {
+ EventHandler eventHandler = readValue(json, EventHandler.class);
+ handlers.add(eventHandler);
+ });
+ recordRedisDaoRequests("getAllEventHandlers");
+ return handlers;
+ }
+
+ private void index(EventHandler eventHandler) {
+ String event = eventHandler.getEvent();
+ String key = nsKey(EVENT_HANDLERS_BY_EVENT, event);
+ orkesJedisProxy.sadd(key, eventHandler.getName());
+ }
+
+ private void removeIndex(EventHandler eventHandler) {
+ String event = eventHandler.getEvent();
+ String key = nsKey(EVENT_HANDLERS_BY_EVENT, event);
+ orkesJedisProxy.srem(key, eventHandler.getName());
+ }
+
+ @Override
+ public List getEventHandlersForEvent(String event, boolean activeOnly) {
+ String key = nsKey(EVENT_HANDLERS_BY_EVENT, event);
+ Set names = orkesJedisProxy.smembers(key);
+ List handlers = new LinkedList<>();
+ for (String name : names) {
+ try {
+ EventHandler eventHandler = getEventHandler(name);
+ recordRedisDaoEventRequests("getEventHandler", event);
+ if (eventHandler.getEvent().equals(event)
+ && (!activeOnly || eventHandler.isActive())) {
+ handlers.add(eventHandler);
+ }
+ } catch (NotFoundException nfe) {
+ LOGGER.info("No matching event handler found for event: {}", event);
+ throw nfe;
+ }
+ }
+ return handlers;
+ }
+
+ private EventHandler getEventHandler(String name) {
+ EventHandler eventHandler = null;
+ String json = orkesJedisProxy.hget(nsKey(EVENT_HANDLERS), name);
+ if (json != null) {
+ eventHandler = readValue(json, EventHandler.class);
+ }
+ return eventHandler;
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisExecutionDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisExecutionDAO.java
new file mode 100644
index 0000000..8f2ab2e
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisExecutionDAO.java
@@ -0,0 +1,730 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.common.metadata.events.EventExecution;
+import com.netflix.conductor.common.metadata.tasks.TaskDef;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.dao.ConcurrentExecutionLimitDAO;
+import com.netflix.conductor.dao.ExecutionDAO;
+import com.netflix.conductor.metrics.Monitors;
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.model.WorkflowModel;
+import com.netflix.conductor.redis.config.AnyRedisCondition;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+@Component
+@Conditional(AnyRedisCondition.class)
+public class RedisExecutionDAO extends BaseDynoDAO
+ implements ExecutionDAO, ConcurrentExecutionLimitDAO {
+
+ public static final Logger LOGGER = LoggerFactory.getLogger(RedisExecutionDAO.class);
+
+ // Keys Families
+ private static final String TASK_LIMIT_BUCKET = "TASK_LIMIT_BUCKET";
+ private static final String IN_PROGRESS_TASKS = "IN_PROGRESS_TASKS";
+ private static final String TASKS_IN_PROGRESS_STATUS =
+ "TASKS_IN_PROGRESS_STATUS"; // Tasks which are in IN_PROGRESS status.
+ private static final String WORKFLOW_TO_TASKS = "WORKFLOW_TO_TASKS";
+ private static final String SCHEDULED_TASKS = "SCHEDULED_TASKS";
+ private static final String TASK = "TASK";
+ private static final String WORKFLOW = "WORKFLOW";
+ private static final String PENDING_WORKFLOWS = "PENDING_WORKFLOWS";
+ private static final String WORKFLOW_DEF_TO_WORKFLOWS = "WORKFLOW_DEF_TO_WORKFLOWS";
+ private static final String EVENT_EXECUTION = "EVENT_EXECUTION";
+ private final int ttlEventExecutionSeconds;
+
+ public RedisExecutionDAO(
+ OrkesJedisProxy orkesJedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties) {
+ super(orkesJedisProxy, objectMapper, conductorProperties, properties);
+
+ ttlEventExecutionSeconds = (int) properties.getEventExecutionPersistenceTTL().getSeconds();
+ }
+
+ private static String dateStr(Long timeInMs) {
+ Date date = new Date(timeInMs);
+ return dateStr(date);
+ }
+
+ private static String dateStr(Date date) {
+ SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
+ return format.format(date);
+ }
+
+ private static List dateStrBetweenDates(Long startdatems, Long enddatems) {
+ List dates = new ArrayList<>();
+ Calendar calendar = new GregorianCalendar();
+ Date startdate = new Date(startdatems);
+ Date enddate = new Date(enddatems);
+ calendar.setTime(startdate);
+ while (calendar.getTime().before(enddate) || calendar.getTime().equals(enddate)) {
+ Date result = calendar.getTime();
+ dates.add(dateStr(result));
+ calendar.add(Calendar.DATE, 1);
+ }
+ return dates;
+ }
+
+ @Override
+ public List getPendingTasksByWorkflow(String taskName, String workflowId) {
+ List tasks = new LinkedList<>();
+
+ List pendingTasks = getPendingTasksForTaskType(taskName);
+ pendingTasks.forEach(
+ pendingTask -> {
+ if (pendingTask.getWorkflowInstanceId().equals(workflowId)) {
+ tasks.add(pendingTask);
+ }
+ });
+
+ return tasks;
+ }
+
+ @Override
+ public List getTasks(String taskDefName, String startKey, int count) {
+ List tasks = new LinkedList<>();
+
+ List pendingTasks = getPendingTasksForTaskType(taskDefName);
+ boolean startKeyFound = startKey == null;
+ int foundcount = 0;
+ for (TaskModel pendingTask : pendingTasks) {
+ if (!startKeyFound) {
+ if (pendingTask.getTaskId().equals(startKey)) {
+ startKeyFound = true;
+ if (startKey != null) {
+ continue;
+ }
+ }
+ }
+ if (startKeyFound && foundcount < count) {
+ tasks.add(pendingTask);
+ foundcount++;
+ }
+ }
+ return tasks;
+ }
+
+ @Override
+ public List createTasks(List tasks) {
+
+ List tasksCreated = new LinkedList<>();
+
+ for (TaskModel task : tasks) {
+ validate(task);
+ String taskKey = task.getReferenceTaskName() + "" + task.getRetryCount();
+ Long added =
+ orkesJedisProxy.hset(
+ nsKey(SCHEDULED_TASKS, task.getWorkflowInstanceId()),
+ taskKey,
+ task.getTaskId());
+ if (added < 1) {
+ LOGGER.debug(
+ "Task already scheduled, skipping the run "
+ + task.getTaskId()
+ + ", ref="
+ + task.getReferenceTaskName()
+ + ", key="
+ + taskKey);
+ continue;
+ }
+
+ if (task.getStatus() != null
+ && !task.getStatus().isTerminal()
+ && task.getScheduledTime() == 0) {
+ task.setScheduledTime(System.currentTimeMillis());
+ }
+
+ correlateTaskToWorkflowInDS(task.getTaskId(), task.getWorkflowInstanceId());
+ LOGGER.debug(
+ "Scheduled task added to WORKFLOW_TO_TASKS workflowId: {}, taskId: {}, taskType: {} during createTasks",
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType());
+
+ String inProgressTaskKey = nsKey(IN_PROGRESS_TASKS, task.getTaskDefName());
+ orkesJedisProxy.sadd(inProgressTaskKey, task.getTaskId());
+ LOGGER.debug(
+ "Scheduled task added to IN_PROGRESS_TASKS with inProgressTaskKey: {}, workflowId: {}, taskId: {}, taskType: {} during createTasks",
+ inProgressTaskKey,
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType());
+
+ updateTask(task);
+ tasksCreated.add(task);
+ }
+
+ return tasksCreated;
+ }
+
+ @Override
+ public void updateTask(TaskModel task) {
+ Optional taskDefinition = task.getTaskDefinition();
+
+ if (taskDefinition.isPresent() && taskDefinition.get().concurrencyLimit() > 0) {
+
+ if (task.getStatus() != null && task.getStatus().equals(TaskModel.Status.IN_PROGRESS)) {
+ orkesJedisProxy.sadd(
+ nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId());
+ LOGGER.debug(
+ "Workflow Task added to TASKS_IN_PROGRESS_STATUS with tasksInProgressKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask",
+ nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName(), task.getTaskId()),
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType(),
+ task.getStatus().name());
+ } else {
+ orkesJedisProxy.srem(
+ nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId());
+ LOGGER.debug(
+ "Workflow Task removed from TASKS_IN_PROGRESS_STATUS with tasksInProgressKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask",
+ nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName(), task.getTaskId()),
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType(),
+ task.getStatus().name());
+ String key = nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName());
+ orkesJedisProxy.zrem(key, task.getTaskId());
+ LOGGER.debug(
+ "Workflow Task removed from TASK_LIMIT_BUCKET with taskLimitBucketKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask",
+ key,
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType(),
+ task.getStatus().name());
+ }
+ }
+
+ String payload = toJson(task);
+ recordRedisDaoPayloadSize(
+ "updateTask",
+ payload.length(),
+ taskDefinition.map(TaskDef::getName).orElse("n/a"),
+ task.getWorkflowType());
+
+ orkesJedisProxy.set(nsKey(TASK, task.getTaskId()), payload);
+ LOGGER.debug(
+ "Workflow task payload saved to TASK with taskKey: {}, workflowId: {}, taskId: {}, taskType: {} during updateTask",
+ nsKey(TASK, task.getTaskId()),
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType());
+ if (task.getStatus() != null && task.getStatus().isTerminal()) {
+ orkesJedisProxy.srem(nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getTaskId());
+ LOGGER.debug(
+ "Workflow Task removed from TASKS_IN_PROGRESS_STATUS with tasksInProgressKey: {}, workflowId: {}, taskId: {}, taskType: {}, taskStatus: {} during updateTask",
+ nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()),
+ task.getWorkflowInstanceId(),
+ task.getTaskId(),
+ task.getTaskType(),
+ task.getStatus().name());
+ }
+
+ Set taskIds =
+ orkesJedisProxy.smembers(nsKey(WORKFLOW_TO_TASKS, task.getWorkflowInstanceId()));
+ if (!taskIds.contains(task.getTaskId())) {
+ correlateTaskToWorkflowInDS(task.getTaskId(), task.getWorkflowInstanceId());
+ }
+ }
+
+ @Override
+ public boolean exceedsLimit(TaskModel task) {
+ Optional taskDefinition = task.getTaskDefinition();
+ if (taskDefinition.isEmpty()) {
+ return false;
+ }
+ int limit = taskDefinition.get().concurrencyLimit();
+ if (limit <= 0) {
+ return false;
+ }
+
+ long current = getInProgressTaskCount(task.getTaskDefName());
+ if (current >= limit) {
+ LOGGER.info(
+ "Task execution count limited. task - {}:{}, limit: {}, current: {}",
+ task.getTaskId(),
+ task.getTaskDefName(),
+ limit,
+ current);
+ Monitors.recordTaskConcurrentExecutionLimited(task.getTaskDefName(), limit);
+ return true;
+ }
+
+ String rateLimitKey = nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName());
+ double score = System.currentTimeMillis();
+ String taskId = task.getTaskId();
+ orkesJedisProxy.zaddnx(rateLimitKey, score, taskId);
+
+ Set ids = orkesJedisProxy.zrangeByScore(rateLimitKey, 0, score + 1, limit);
+ boolean rateLimited = !ids.contains(taskId);
+ if (rateLimited) {
+ LOGGER.info(
+ "Task execution count limited. task - {}:{}, limit: {}, current: {}",
+ task.getTaskId(),
+ task.getTaskDefName(),
+ limit,
+ current);
+ String inProgressKey = nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName());
+ // Cleanup any items that are still present in the rate limit bucket but not in progress
+ // anymore!
+ ids.stream()
+ .filter(id -> !orkesJedisProxy.sismember(inProgressKey, id))
+ .forEach(id2 -> orkesJedisProxy.zrem(rateLimitKey, id2));
+ Monitors.recordTaskRateLimited(task.getTaskDefName(), limit);
+ }
+ return rateLimited;
+ }
+
+ private void removeTaskMappings(TaskModel task) {
+ String taskKey = task.getReferenceTaskName() + "" + task.getRetryCount();
+
+ orkesJedisProxy.hdel(nsKey(SCHEDULED_TASKS, task.getWorkflowInstanceId()), taskKey);
+ orkesJedisProxy.srem(nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getTaskId());
+ orkesJedisProxy.srem(
+ nsKey(WORKFLOW_TO_TASKS, task.getWorkflowInstanceId()), task.getTaskId());
+ orkesJedisProxy.srem(
+ nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId());
+ orkesJedisProxy.zrem(nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName()), task.getTaskId());
+ }
+
+ private void removeTaskMappingsWithExpiry(TaskModel task) {
+ String taskKey = task.getReferenceTaskName() + "" + task.getRetryCount();
+
+ orkesJedisProxy.hdel(nsKey(SCHEDULED_TASKS, task.getWorkflowInstanceId()), taskKey);
+ orkesJedisProxy.srem(nsKey(IN_PROGRESS_TASKS, task.getTaskDefName()), task.getTaskId());
+ orkesJedisProxy.srem(
+ nsKey(TASKS_IN_PROGRESS_STATUS, task.getTaskDefName()), task.getTaskId());
+ orkesJedisProxy.zrem(nsKey(TASK_LIMIT_BUCKET, task.getTaskDefName()), task.getTaskId());
+ }
+
+ @Override
+ public boolean removeTask(String taskId) {
+ TaskModel task = getTask(taskId);
+ if (task == null) {
+ LOGGER.warn("No such task found by id {}", taskId);
+ return false;
+ }
+ removeTaskMappings(task);
+
+ orkesJedisProxy.del(nsKey(TASK, task.getTaskId()));
+ return true;
+ }
+
+ private boolean removeTaskWithExpiry(String taskId, int ttlSeconds) {
+ TaskModel task = getTask(taskId);
+ if (task == null) {
+ LOGGER.warn("No such task found by id {}", taskId);
+ return false;
+ }
+ removeTaskMappingsWithExpiry(task);
+
+ orkesJedisProxy.expire(nsKey(TASK, task.getTaskId()), ttlSeconds);
+ return true;
+ }
+
+ @Override
+ public TaskModel getTask(String taskId) {
+ Preconditions.checkNotNull(taskId, "taskId cannot be null");
+ return Optional.ofNullable(orkesJedisProxy.get(nsKey(TASK, taskId)))
+ .map(
+ json -> {
+ TaskModel task = readValue(json, TaskModel.class);
+ recordRedisDaoPayloadSize(
+ "getTask",
+ toJson(task).length(),
+ task.getTaskType(),
+ task.getWorkflowType());
+ return task;
+ })
+ .orElse(null);
+ }
+
+ @Override
+ public List getTasks(List taskIds) {
+ return taskIds.stream()
+ .map(taskId -> nsKey(TASK, taskId))
+ .map(orkesJedisProxy::get)
+ .filter(Objects::nonNull)
+ .map(
+ jsonString -> {
+ TaskModel task = readValue(jsonString, TaskModel.class);
+ return task;
+ })
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List getTasksForWorkflow(String workflowId) {
+ Preconditions.checkNotNull(workflowId, "workflowId cannot be null");
+ Set taskIds = orkesJedisProxy.smembers(nsKey(WORKFLOW_TO_TASKS, workflowId));
+ return getTasks(new ArrayList<>(taskIds));
+ }
+
+ @Override
+ public List getPendingTasksForTaskType(String taskName) {
+ Preconditions.checkNotNull(taskName, "task name cannot be null");
+ Set taskIds = orkesJedisProxy.smembers(nsKey(IN_PROGRESS_TASKS, taskName));
+ return getTasks(new ArrayList<>(taskIds));
+ }
+
+ @Override
+ public String createWorkflow(WorkflowModel workflow) {
+ return insertOrUpdateWorkflow(workflow, false);
+ }
+
+ @Override
+ public String updateWorkflow(WorkflowModel workflow) {
+ return insertOrUpdateWorkflow(workflow, true);
+ }
+
+ @Override
+ public boolean removeWorkflow(String workflowId) {
+ WorkflowModel workflow = getWorkflow(workflowId, true);
+ if (workflow != null) {
+ // Remove from lists
+ String key =
+ nsKey(
+ WORKFLOW_DEF_TO_WORKFLOWS,
+ workflow.getWorkflowName(),
+ dateStr(workflow.getCreateTime()));
+ orkesJedisProxy.srem(key, workflowId);
+ orkesJedisProxy.srem(nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflowId);
+
+ // Remove the object
+ orkesJedisProxy.del(nsKey(WORKFLOW, workflowId));
+ for (TaskModel task : workflow.getTasks()) {
+ removeTask(task.getTaskId());
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public boolean removeWorkflowWithExpiry(String workflowId, int ttlSeconds) {
+ WorkflowModel workflow = getWorkflow(workflowId, true);
+ if (workflow != null) {
+ // Remove from lists
+ String key =
+ nsKey(
+ WORKFLOW_DEF_TO_WORKFLOWS,
+ workflow.getWorkflowName(),
+ dateStr(workflow.getCreateTime()));
+ orkesJedisProxy.srem(key, workflowId);
+ orkesJedisProxy.srem(nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflowId);
+
+ // Remove the object
+ orkesJedisProxy.expire(nsKey(WORKFLOW, workflowId), ttlSeconds);
+ for (TaskModel task : workflow.getTasks()) {
+ removeTaskWithExpiry(task.getTaskId(), ttlSeconds);
+ }
+ orkesJedisProxy.expire(nsKey(WORKFLOW_TO_TASKS, workflowId), ttlSeconds);
+
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void removeFromPendingWorkflow(String workflowType, String workflowId) {
+ orkesJedisProxy.del(nsKey(SCHEDULED_TASKS, workflowId));
+ orkesJedisProxy.srem(nsKey(PENDING_WORKFLOWS, workflowType), workflowId);
+ }
+
+ @Override
+ public WorkflowModel getWorkflow(String workflowId) {
+ return getWorkflow(workflowId, true);
+ }
+
+ @Override
+ public WorkflowModel getWorkflow(String workflowId, boolean includeTasks) {
+ String json = orkesJedisProxy.get(nsKey(WORKFLOW, workflowId));
+ WorkflowModel workflow = null;
+
+ if (json != null) {
+ workflow = readValue(json, WorkflowModel.class);
+ if (includeTasks) {
+ List tasks = getTasksForWorkflow(workflowId);
+ tasks.sort(Comparator.comparingInt(TaskModel::getSeq));
+ workflow.setTasks(tasks);
+ }
+ }
+ return workflow;
+ }
+
+ /**
+ * @param workflowName name of the workflow
+ * @param version the workflow version
+ * @return list of workflow ids that are in RUNNING state returns workflows of all versions
+ * for the given workflow name
+ */
+ @Override
+ public List getRunningWorkflowIds(String workflowName, int version) {
+ Preconditions.checkNotNull(workflowName, "workflowName cannot be null");
+ List workflowIds;
+ Set pendingWorkflows =
+ orkesJedisProxy.smembers(nsKey(PENDING_WORKFLOWS, workflowName));
+ workflowIds = new LinkedList<>(pendingWorkflows);
+ return workflowIds;
+ }
+
+ /**
+ * @param workflowName name of the workflow
+ * @param version the workflow version
+ * @return list of workflows that are in RUNNING state
+ */
+ @Override
+ public List getPendingWorkflowsByType(String workflowName, int version) {
+ Preconditions.checkNotNull(workflowName, "workflowName cannot be null");
+ List workflowIds = getRunningWorkflowIds(workflowName, version);
+ return workflowIds.stream()
+ .map(this::getWorkflow)
+ .filter(workflow -> workflow.getWorkflowVersion() == version)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public List getWorkflowsByType(
+ String workflowName, Long startTime, Long endTime) {
+ Preconditions.checkNotNull(workflowName, "workflowName cannot be null");
+ Preconditions.checkNotNull(startTime, "startTime cannot be null");
+ Preconditions.checkNotNull(endTime, "endTime cannot be null");
+
+ List workflows = new LinkedList<>();
+
+ // Get all date strings between start and end
+ List dateStrs = dateStrBetweenDates(startTime, endTime);
+ dateStrs.forEach(
+ dateStr -> {
+ String key = nsKey(WORKFLOW_DEF_TO_WORKFLOWS, workflowName, dateStr);
+ orkesJedisProxy
+ .smembers(key)
+ .forEach(
+ workflowId -> {
+ try {
+ WorkflowModel workflow = getWorkflow(workflowId);
+ if (workflow.getCreateTime() >= startTime
+ && workflow.getCreateTime() <= endTime) {
+ workflows.add(workflow);
+ }
+ } catch (Exception e) {
+ LOGGER.error(
+ "Failed to get workflow: {}", workflowId, e);
+ }
+ });
+ });
+
+ return workflows;
+ }
+
+ @Override
+ public List getWorkflowsByCorrelationId(
+ String workflowName, String correlationId, boolean includeTasks) {
+ throw new UnsupportedOperationException(
+ "This method is not implemented in RedisExecutionDAO. Please use ExecutionDAOFacade instead.");
+ }
+
+ @Override
+ public boolean canSearchAcrossWorkflows() {
+ return false;
+ }
+
+ /**
+ * Inserts a new workflow/ updates an existing workflow in the datastore. Additionally, if a
+ * workflow is in terminal state, it is removed from the set of pending workflows.
+ *
+ * @param workflow the workflow instance
+ * @param update flag to identify if update or create operation
+ * @return the workflowId
+ */
+ private String insertOrUpdateWorkflow(WorkflowModel workflow, boolean update) {
+ Preconditions.checkNotNull(workflow, "workflow object cannot be null");
+
+ List tasks = workflow.getTasks();
+ workflow.setTasks(new LinkedList<>());
+
+ String payload = toJson(workflow);
+ // Store the workflow object
+ orkesJedisProxy.set(nsKey(WORKFLOW, workflow.getWorkflowId()), payload);
+ recordRedisDaoPayloadSize(
+ "storeWorkflow", payload.length(), "n/a", workflow.getWorkflowName());
+ if (!update) {
+ // Add to list of workflows for a workflowdef
+ String key =
+ nsKey(
+ WORKFLOW_DEF_TO_WORKFLOWS,
+ workflow.getWorkflowName(),
+ dateStr(workflow.getCreateTime()));
+ orkesJedisProxy.sadd(key, workflow.getWorkflowId());
+ }
+ // Add or remove from the pending workflows
+ if (workflow.getStatus().isTerminal()) {
+ orkesJedisProxy.srem(
+ nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflow.getWorkflowId());
+ } else {
+ orkesJedisProxy.sadd(
+ nsKey(PENDING_WORKFLOWS, workflow.getWorkflowName()), workflow.getWorkflowId());
+ }
+
+ workflow.setTasks(tasks);
+ return workflow.getWorkflowId();
+ }
+
+ /**
+ * Stores the correlation of a task to the workflow instance in the datastore
+ *
+ * @param taskId the taskId to be correlated
+ * @param workflowInstanceId the workflowId to which the tasks belongs to
+ */
+ @VisibleForTesting
+ void correlateTaskToWorkflowInDS(String taskId, String workflowInstanceId) {
+ String workflowToTaskKey = nsKey(WORKFLOW_TO_TASKS, workflowInstanceId);
+ orkesJedisProxy.sadd(workflowToTaskKey, taskId);
+ LOGGER.debug(
+ "Task mapped in WORKFLOW_TO_TASKS with workflowToTaskKey: {}, workflowId: {}, taskId: {}",
+ workflowToTaskKey,
+ workflowInstanceId,
+ taskId);
+ }
+
+ public long getPendingWorkflowCount(String workflowName) {
+ String key = nsKey(PENDING_WORKFLOWS, workflowName);
+ return orkesJedisProxy.scard(key);
+ }
+
+ @Override
+ public long getInProgressTaskCount(String taskDefName) {
+ String inProgressKey = nsKey(TASKS_IN_PROGRESS_STATUS, taskDefName);
+ return orkesJedisProxy.scard(inProgressKey);
+ }
+
+ @Override
+ public boolean addEventExecution(EventExecution eventExecution) {
+ try {
+ String key =
+ nsKey(
+ EVENT_EXECUTION,
+ eventExecution.getName(),
+ eventExecution.getEvent(),
+ eventExecution.getMessageId());
+ String json = objectMapper.writeValueAsString(eventExecution);
+ recordRedisDaoEventRequests("addEventExecution", eventExecution.getEvent());
+ recordRedisDaoPayloadSize(
+ "addEventExecution", json.length(), eventExecution.getEvent(), "n/a");
+ boolean added = orkesJedisProxy.hsetnx(key, eventExecution.getId(), json) == 1L;
+
+ if (ttlEventExecutionSeconds > 0) {
+ orkesJedisProxy.expire(key, ttlEventExecutionSeconds);
+ }
+
+ return added;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Unable to add event execution for " + eventExecution.getId(), e);
+ }
+ }
+
+ @Override
+ public void updateEventExecution(EventExecution eventExecution) {
+ try {
+
+ String key =
+ nsKey(
+ EVENT_EXECUTION,
+ eventExecution.getName(),
+ eventExecution.getEvent(),
+ eventExecution.getMessageId());
+ String json = objectMapper.writeValueAsString(eventExecution);
+ LOGGER.info("updating event execution {}", key);
+ orkesJedisProxy.hset(key, eventExecution.getId(), json);
+ recordRedisDaoEventRequests("updateEventExecution", eventExecution.getEvent());
+ recordRedisDaoPayloadSize(
+ "updateEventExecution", json.length(), eventExecution.getEvent(), "n/a");
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Unable to update event execution for " + eventExecution.getId(), e);
+ }
+ }
+
+ @Override
+ public void removeEventExecution(EventExecution eventExecution) {
+ try {
+ String key =
+ nsKey(
+ EVENT_EXECUTION,
+ eventExecution.getName(),
+ eventExecution.getEvent(),
+ eventExecution.getMessageId());
+ LOGGER.info("removing event execution {}", key);
+ orkesJedisProxy.hdel(key, eventExecution.getId());
+ recordRedisDaoEventRequests("removeEventExecution", eventExecution.getEvent());
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Unable to remove event execution for " + eventExecution.getId(), e);
+ }
+ }
+
+ public List getEventExecutions(
+ String eventHandlerName, String eventName, String messageId, int max) {
+ try {
+ String key = nsKey(EVENT_EXECUTION, eventHandlerName, eventName, messageId);
+ LOGGER.info("getting event execution {}", key);
+ List executions = new LinkedList<>();
+ for (int i = 0; i < max; i++) {
+ String field = messageId + "_" + i;
+ String value = orkesJedisProxy.hget(key, field);
+ if (value == null) {
+ break;
+ }
+ recordRedisDaoEventRequests("getEventExecution", eventHandlerName);
+ recordRedisDaoPayloadSize(
+ "getEventExecution", value.length(), eventHandlerName, "n/a");
+ EventExecution eventExecution = objectMapper.readValue(value, EventExecution.class);
+ executions.add(eventExecution);
+ }
+ return executions;
+
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to get event execution for " + e);
+ }
+ }
+
+ private void validate(TaskModel task) {
+ try {
+ Preconditions.checkNotNull(task, "task object cannot be null");
+ Preconditions.checkNotNull(task.getTaskId(), "Task id cannot be null");
+ Preconditions.checkNotNull(
+ task.getWorkflowInstanceId(), "Workflow instance id cannot be null");
+ Preconditions.checkNotNull(
+ task.getReferenceTaskName(), "Task reference name cannot be null");
+ } catch (NullPointerException npe) {
+ throw new IllegalArgumentException(npe);
+ }
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisMetadataDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisMetadataDAO.java
new file mode 100644
index 0000000..c5f5572
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisMetadataDAO.java
@@ -0,0 +1,306 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.netflix.conductor.common.metadata.Auditable;
+import com.netflix.conductor.common.metadata.tasks.TaskDef;
+import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.core.exception.ConflictException;
+import com.netflix.conductor.core.exception.NotFoundException;
+import com.netflix.conductor.dao.MetadataDAO;
+import com.netflix.conductor.metrics.Monitors;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+
+import static java.util.Objects.isNull;
+
+public class RedisMetadataDAO extends BaseDynoDAO implements MetadataDAO {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RedisMetadataDAO.class);
+
+ // Keys Families
+ private static final String ALL_TASK_DEFS = "TASK_DEFS";
+ private static final String WORKFLOW_DEF_NAMES = "WORKFLOW_DEF_NAMES";
+ private static final String WORKFLOW_DEF = "WORKFLOW_DEF";
+ private static final String LATEST = "latest";
+ private static final String className = RedisMetadataDAO.class.getSimpleName();
+ private Map taskDefCache = new HashMap<>();
+
+ RedisMetadataDAO(
+ OrkesJedisProxy orkesJedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties) {
+ super(orkesJedisProxy, objectMapper, conductorProperties, properties);
+ refreshTaskDefs();
+ long cacheRefreshTime = properties.getTaskDefCacheRefreshInterval().getSeconds();
+ if (cacheRefreshTime > 0) {
+ Executors.newSingleThreadScheduledExecutor()
+ .scheduleWithFixedDelay(
+ this::refreshTaskDefs,
+ cacheRefreshTime,
+ cacheRefreshTime,
+ TimeUnit.SECONDS);
+ }
+ }
+
+ @Override
+ public TaskDef createTaskDef(TaskDef taskDef) {
+ insertOrUpdateTaskDef(taskDef);
+ return taskDef;
+ }
+
+ @Override
+ public TaskDef updateTaskDef(TaskDef taskDef) {
+ insertOrUpdateTaskDef(taskDef);
+ return taskDef;
+ }
+
+ private String insertOrUpdateTaskDef(TaskDef taskDef) {
+ // Store all task def in under one key
+ String payload = toJson(taskDef);
+ orkesJedisProxy.hset(nsKey(ALL_TASK_DEFS), taskDef.getName(), payload);
+ refreshTaskDefs();
+ return taskDef.getName();
+ }
+
+ private void refreshTaskDefs() {
+ try {
+ Map map = new HashMap<>();
+ getAllTaskDefs().forEach(taskDef -> map.put(taskDef.getName(), taskDef));
+ this.taskDefCache = map;
+ LOGGER.debug("Refreshed task defs " + this.taskDefCache.size());
+ } catch (Exception e) {
+ Monitors.error(className, "refreshTaskDefs");
+ LOGGER.error("refresh TaskDefs failed ", e);
+ }
+ }
+
+ @Override
+ public TaskDef getTaskDef(String name) {
+ return Optional.ofNullable(taskDefCache.get(name)).orElseGet(() -> getTaskDefFromDB(name));
+ }
+
+ private TaskDef getTaskDefFromDB(String name) {
+ Preconditions.checkNotNull(name, "TaskDef name cannot be null");
+
+ TaskDef taskDef = null;
+ String taskDefJsonStr = orkesJedisProxy.hget(nsKey(ALL_TASK_DEFS), name);
+ if (taskDefJsonStr != null) {
+ taskDef = readValue(taskDefJsonStr, TaskDef.class);
+ recordRedisDaoRequests("getTaskDef");
+ recordRedisDaoPayloadSize(
+ "getTaskDef", taskDefJsonStr.length(), taskDef.getName(), "n/a");
+ }
+ return taskDef;
+ }
+
+ @Override
+ public List getAllTaskDefs() {
+ List allTaskDefs = new LinkedList<>();
+
+ recordRedisDaoRequests("getAllTaskDefs");
+ Map taskDefs = orkesJedisProxy.hgetAll(nsKey(ALL_TASK_DEFS));
+ int size = 0;
+ if (taskDefs.size() > 0) {
+ for (String taskDefJsonStr : taskDefs.values()) {
+ if (taskDefJsonStr != null) {
+ allTaskDefs.add(readValue(taskDefJsonStr, TaskDef.class));
+ size += taskDefJsonStr.length();
+ }
+ }
+ recordRedisDaoPayloadSize("getAllTaskDefs", size, "n/a", "n/a");
+ }
+
+ return allTaskDefs;
+ }
+
+ @Override
+ public void removeTaskDef(String name) {
+ Preconditions.checkNotNull(name, "TaskDef name cannot be null");
+ Long result = orkesJedisProxy.hdel(nsKey(ALL_TASK_DEFS), name);
+ if (!result.equals(1L)) {
+ throw new NotFoundException("Cannot remove the task - no such task definition");
+ }
+ recordRedisDaoRequests("removeTaskDef");
+ refreshTaskDefs();
+ }
+
+ @Override
+ public void createWorkflowDef(WorkflowDef def) {
+ if (orkesJedisProxy.hexists(
+ nsKey(WORKFLOW_DEF, def.getName()), String.valueOf(def.getVersion()))) {
+ throw new ConflictException("Workflow with " + def.key() + " already exists!");
+ }
+ _createOrUpdate(def);
+ }
+
+ @Override
+ public void updateWorkflowDef(WorkflowDef def) {
+ _createOrUpdate(def);
+ }
+
+ @Override
+ /*
+ * @param name Name of the workflow definition
+ * @return Latest version of workflow definition
+ * @see WorkflowDef
+ */
+ public Optional getLatestWorkflowDef(String name) {
+ Preconditions.checkNotNull(name, "WorkflowDef name cannot be null");
+ WorkflowDef workflowDef = null;
+
+ Optional optionalMaxVersion = getWorkflowMaxVersion(name);
+
+ if (optionalMaxVersion.isPresent()) {
+ String latestdata =
+ orkesJedisProxy.hget(
+ nsKey(WORKFLOW_DEF, name), optionalMaxVersion.get().toString());
+ if (latestdata != null) {
+ workflowDef = readValue(latestdata, WorkflowDef.class);
+ }
+ }
+
+ return Optional.ofNullable(workflowDef);
+ }
+
+ private Optional getWorkflowMaxVersion(String workflowName) {
+ return orkesJedisProxy.hkeys(nsKey(WORKFLOW_DEF, workflowName)).stream()
+ .filter(key -> !key.equals(LATEST))
+ .map(Integer::valueOf)
+ .max(Comparator.naturalOrder());
+ }
+
+ public List getAllVersions(String name) {
+ Preconditions.checkNotNull(name, "WorkflowDef name cannot be null");
+ List workflows = new LinkedList<>();
+
+ recordRedisDaoRequests("getAllWorkflowDefsByName");
+ Map workflowDefs = orkesJedisProxy.hgetAll(nsKey(WORKFLOW_DEF, name));
+ int size = 0;
+ for (String key : workflowDefs.keySet()) {
+ if (key.equals(LATEST)) {
+ continue;
+ }
+ String workflowDef = workflowDefs.get(key);
+ workflows.add(readValue(workflowDef, WorkflowDef.class));
+ size += workflowDef.length();
+ }
+ recordRedisDaoPayloadSize("getAllWorkflowDefsByName", size, "n/a", name);
+
+ return workflows;
+ }
+
+ @Override
+ public Optional getWorkflowDef(String name, int version) {
+ Preconditions.checkNotNull(name, "WorkflowDef name cannot be null");
+ WorkflowDef def = null;
+
+ recordRedisDaoRequests("getWorkflowDef");
+ String workflowDefJsonString =
+ orkesJedisProxy.hget(nsKey(WORKFLOW_DEF, name), String.valueOf(version));
+ if (workflowDefJsonString != null) {
+ def = readValue(workflowDefJsonString, WorkflowDef.class);
+ recordRedisDaoPayloadSize(
+ "getWorkflowDef", workflowDefJsonString.length(), "n/a", name);
+ }
+ return Optional.ofNullable(def);
+ }
+
+ @Override
+ public void removeWorkflowDef(String name, Integer version) {
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(name), "WorkflowDef name cannot be null");
+ Preconditions.checkNotNull(version, "Input version cannot be null");
+ Long result = orkesJedisProxy.hdel(nsKey(WORKFLOW_DEF, name), String.valueOf(version));
+ if (!result.equals(1L)) {
+ throw new NotFoundException(
+ String.format(
+ "Cannot remove the workflow - no such workflow"
+ + " definition: %s version: %d",
+ name, version));
+ }
+
+ // check if there are any more versions remaining if not delete the
+ // workflow name
+ Optional optionMaxVersion = getWorkflowMaxVersion(name);
+
+ // delete workflow name
+ if (!optionMaxVersion.isPresent()) {
+ orkesJedisProxy.srem(nsKey(WORKFLOW_DEF_NAMES), name);
+ }
+
+ recordRedisDaoRequests("removeWorkflowDef");
+ }
+
+ public List findAll() {
+ Set wfNames = orkesJedisProxy.smembers(nsKey(WORKFLOW_DEF_NAMES));
+ return new ArrayList<>(wfNames);
+ }
+
+ @Override
+ public List getAllWorkflowDefs() {
+ List workflows = new LinkedList<>();
+
+ // Get all from WORKFLOW_DEF_NAMES
+ recordRedisDaoRequests("getAllWorkflowDefs");
+ Set wfNames = orkesJedisProxy.smembers(nsKey(WORKFLOW_DEF_NAMES));
+ int size = 0;
+ for (String wfName : wfNames) {
+ Map workflowDefs = orkesJedisProxy.hgetAll(nsKey(WORKFLOW_DEF, wfName));
+ for (String key : workflowDefs.keySet()) {
+ if (key.equals(LATEST)) {
+ continue;
+ }
+ String workflowDef = workflowDefs.get(key);
+ workflows.add(readValue(workflowDef, WorkflowDef.class));
+ size += workflowDef.length();
+ }
+ }
+ recordRedisDaoPayloadSize("getAllWorkflowDefs", size, "n/a", "n/a");
+ return workflows;
+ }
+
+ private void _createOrUpdate(WorkflowDef workflowDef) {
+ if (isNull(workflowDef.getUpdateTime())) {
+ workflowDef.setUpdateTime(System.currentTimeMillis());
+ }
+ if (isNull(workflowDef.getCreateTime())) {
+ workflowDef.setCreateTime(
+ getWorkflowDef(workflowDef.getName(), workflowDef.getVersion())
+ .map(Auditable::getCreateTime)
+ .orElse(System.currentTimeMillis()));
+ }
+
+ // First set the workflow def
+ orkesJedisProxy.hset(
+ nsKey(WORKFLOW_DEF, workflowDef.getName()),
+ String.valueOf(workflowDef.getVersion()),
+ toJson(workflowDef));
+
+ orkesJedisProxy.sadd(nsKey(WORKFLOW_DEF_NAMES), workflowDef.getName());
+ recordRedisDaoRequests("storeWorkflowDef", "n/a", workflowDef.getName());
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisPollDataDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisPollDataDAO.java
new file mode 100644
index 0000000..aeb27ac
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisPollDataDAO.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.common.metadata.tasks.PollData;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.dao.PollDataDAO;
+import com.netflix.conductor.redis.config.AnyRedisCondition;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
+import lombok.extern.slf4j.Slf4j;
+
+@Component
+@Conditional(AnyRedisCondition.class)
+@Slf4j
+public class RedisPollDataDAO extends BaseDynoDAO implements PollDataDAO {
+
+ private static final String POLL_DATA = "POLL_DATA";
+
+ private final Clock clock;
+
+ private final Map lastUpdateTimes = new ConcurrentHashMap<>();
+
+ private static final int MAX_UPDATE_FREQUENCY = 10_000; // 10 second
+
+ public RedisPollDataDAO(
+ OrkesJedisProxy orkesJedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties) {
+ super(orkesJedisProxy, objectMapper, conductorProperties, properties);
+ this.clock = Clock.systemDefaultZone();
+ log.info("Using OrkesPollDataDAO");
+ }
+
+ @Override
+ public void updateLastPollData(String taskDefName, String domain, String workerId) {
+
+ long now = clock.millis();
+ lastUpdateTimes.compute(
+ taskDefName,
+ (s, lastUpdateTime) -> {
+ if (lastUpdateTime == null || lastUpdateTime < (now - MAX_UPDATE_FREQUENCY)) {
+ _updateLastPollData(taskDefName, domain, workerId);
+ return now;
+ }
+ return lastUpdateTime;
+ });
+ }
+
+ private void _updateLastPollData(String taskDefName, String domain, String workerId) {
+ Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
+ PollData pollData = new PollData(taskDefName, domain, workerId, System.currentTimeMillis());
+
+ String key = nsKey(POLL_DATA, pollData.getQueueName());
+ String field = (domain == null) ? "DEFAULT" : domain;
+
+ String payload = toJson(pollData);
+ recordRedisDaoRequests("updatePollData");
+ recordRedisDaoPayloadSize("updatePollData", payload.length(), "n/a", "n/a");
+ orkesJedisProxy.hset(key, field, payload);
+ }
+
+ @Override
+ public PollData getPollData(String taskDefName, String domain) {
+ Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
+
+ String key = nsKey(POLL_DATA, taskDefName);
+ String field = (domain == null) ? "DEFAULT" : domain;
+
+ String pollDataJsonString = orkesJedisProxy.hget(key, field);
+ recordRedisDaoRequests("getPollData");
+ recordRedisDaoPayloadSize(
+ "getPollData", StringUtils.length(pollDataJsonString), "n/a", "n/a");
+
+ PollData pollData = null;
+ if (StringUtils.isNotBlank(pollDataJsonString)) {
+ pollData = readValue(pollDataJsonString, PollData.class);
+ }
+ return pollData;
+ }
+
+ @Override
+ public List getPollData(String taskDefName) {
+ Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
+
+ String key = nsKey(POLL_DATA, taskDefName);
+
+ Map pMapdata = orkesJedisProxy.hgetAll(key);
+ List pollData = new ArrayList<>();
+ if (pMapdata != null) {
+ pMapdata.values()
+ .forEach(
+ pollDataJsonString -> {
+ pollData.add(readValue(pollDataJsonString, PollData.class));
+ recordRedisDaoRequests("getPollData");
+ recordRedisDaoPayloadSize(
+ "getPollData", pollDataJsonString.length(), "n/a", "n/a");
+ });
+ }
+ return pollData;
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisRateLimitingDAO.java b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisRateLimitingDAO.java
new file mode 100644
index 0000000..9a5cc49
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dao/RedisRateLimitingDAO.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dao;
+
+import java.util.Optional;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
+
+import com.netflix.conductor.common.metadata.tasks.TaskDef;
+import com.netflix.conductor.core.config.ConductorProperties;
+import com.netflix.conductor.dao.RateLimitingDAO;
+import com.netflix.conductor.metrics.Monitors;
+import com.netflix.conductor.model.TaskModel;
+import com.netflix.conductor.redis.config.AnyRedisCondition;
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.conductor.redis.jedis.OrkesJedisProxy;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+@Component
+@Conditional(AnyRedisCondition.class)
+public class RedisRateLimitingDAO extends BaseDynoDAO implements RateLimitingDAO {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RedisRateLimitingDAO.class);
+
+ private static final String TASK_RATE_LIMIT_BUCKET = "TASK_RATE_LIMIT_BUCKET";
+
+ public RedisRateLimitingDAO(
+ OrkesJedisProxy jedisProxy,
+ ObjectMapper objectMapper,
+ ConductorProperties conductorProperties,
+ RedisProperties properties) {
+ super(jedisProxy, objectMapper, conductorProperties, properties);
+ }
+
+ /**
+ * This method evaluates if the {@link TaskDef} is rate limited or not based on {@link
+ * TaskModel#getRateLimitPerFrequency()} and {@link TaskModel#getRateLimitFrequencyInSeconds()}
+ * if not checks the {@link TaskModel} is rate limited or not based on {@link
+ * TaskModel#getRateLimitPerFrequency()} and {@link TaskModel#getRateLimitFrequencyInSeconds()}
+ *
+ *
The rate limiting is implemented using the Redis constructs of sorted set and TTL of each
+ * element in the rate limited bucket.
+ *
+ *
+ * - All the entries that are in the not in the frequency bucket are cleaned up by
+ * leveraging {@link OrkesJedisProxy#zremrangeByScore(String, String, String)}, this is
+ * done to make the next step of evaluation efficient
+ *
- A current count(tasks executed within the frequency) is calculated based on the current
+ * time and the beginning of the rate limit frequency time(which is current time - {@link
+ * TaskModel#getRateLimitFrequencyInSeconds()} in millis), this is achieved by using
+ * {@link OrkesJedisProxy#zcount(String, double, double)}
+ *
- Once the count is calculated then a evaluation is made to determine if it is within the
+ * bounds of {@link TaskModel#getRateLimitPerFrequency()}, if so the count is increased
+ * and an expiry TTL is added to the entry
+ *
+ *
+ * @param task: which needs to be evaluated whether it is rateLimited or not
+ * @return true: If the {@link TaskModel} is rateLimited false: If the {@link TaskModel} is not
+ * rateLimited
+ */
+ @Override
+ public boolean exceedsRateLimitPerFrequency(TaskModel task, TaskDef taskDef) {
+ // Check if the TaskDefinition is not null then pick the definition values or else pick from
+ // the Task
+ ImmutablePair rateLimitPair =
+ Optional.ofNullable(taskDef)
+ .map(
+ definition ->
+ new ImmutablePair<>(
+ definition.getRateLimitPerFrequency(),
+ definition.getRateLimitFrequencyInSeconds()))
+ .orElse(
+ new ImmutablePair<>(
+ task.getRateLimitPerFrequency(),
+ task.getRateLimitFrequencyInSeconds()));
+
+ int rateLimitPerFrequency = rateLimitPair.getLeft();
+ int rateLimitFrequencyInSeconds = rateLimitPair.getRight();
+ if (rateLimitPerFrequency <= 0 || rateLimitFrequencyInSeconds <= 0) {
+ LOGGER.debug(
+ "Rate limit not applied to the Task: {} either rateLimitPerFrequency: {} or rateLimitFrequencyInSeconds: {} is 0 or less",
+ task,
+ rateLimitPerFrequency,
+ rateLimitFrequencyInSeconds);
+ return false;
+ } else {
+ LOGGER.debug(
+ "Evaluating rate limiting for TaskId: {} with TaskDefinition of: {} with rateLimitPerFrequency: {} and rateLimitFrequencyInSeconds: {}",
+ task.getTaskId(),
+ task.getTaskDefName(),
+ rateLimitPerFrequency,
+ rateLimitFrequencyInSeconds);
+ long currentTimeEpochMillis = System.currentTimeMillis();
+ long currentTimeEpochMinusRateLimitBucket =
+ currentTimeEpochMillis - (rateLimitFrequencyInSeconds * 1000L);
+ String key = nsKey(TASK_RATE_LIMIT_BUCKET, task.getTaskDefName());
+ orkesJedisProxy.zremrangeByScore(
+ key, "-inf", String.valueOf(currentTimeEpochMinusRateLimitBucket));
+ int currentBucketCount =
+ Math.toIntExact(
+ orkesJedisProxy.zcount(
+ key,
+ currentTimeEpochMinusRateLimitBucket,
+ currentTimeEpochMillis));
+ if (currentBucketCount < rateLimitPerFrequency) {
+ orkesJedisProxy.zadd(
+ key, currentTimeEpochMillis, String.valueOf(currentTimeEpochMillis));
+ orkesJedisProxy.expire(key, rateLimitFrequencyInSeconds);
+ LOGGER.info(
+ "TaskId: {} with TaskDefinition of: {} has rateLimitPerFrequency: {} and rateLimitFrequencyInSeconds: {} within the rate limit with current count {}",
+ task.getTaskId(),
+ task.getTaskDefName(),
+ rateLimitPerFrequency,
+ rateLimitFrequencyInSeconds,
+ ++currentBucketCount);
+ Monitors.recordTaskRateLimited(task.getTaskDefName(), rateLimitPerFrequency);
+ return false;
+ } else {
+ LOGGER.info(
+ "TaskId: {} with TaskDefinition of: {} has rateLimitPerFrequency: {} and rateLimitFrequencyInSeconds: {} is out of bounds of rate limit with current count {}",
+ task.getTaskId(),
+ task.getTaskDefName(),
+ rateLimitPerFrequency,
+ rateLimitFrequencyInSeconds,
+ currentBucketCount);
+ return true;
+ }
+ }
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/ConfigurationHostSupplier.java b/persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/ConfigurationHostSupplier.java
new file mode 100644
index 0000000..296212a
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/ConfigurationHostSupplier.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dynoqueue;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.dyno.connectionpool.Host;
+import com.netflix.dyno.connectionpool.HostBuilder;
+import com.netflix.dyno.connectionpool.HostSupplier;
+
+public class ConfigurationHostSupplier implements HostSupplier {
+
+ private static final Logger log = LoggerFactory.getLogger(ConfigurationHostSupplier.class);
+
+ private final RedisProperties properties;
+
+ public ConfigurationHostSupplier(RedisProperties properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public List getHosts() {
+ return parseHostsFromConfig();
+ }
+
+ private List parseHostsFromConfig() {
+ String hosts = properties.getHosts();
+ if (hosts == null) {
+ // FIXME This type of validation probably doesn't belong here.
+ String message =
+ "Missing dynomite/redis hosts. Ensure 'workflow.dynomite.cluster.hosts' has been set in the supplied configuration.";
+ log.error(message);
+ throw new RuntimeException(message);
+ }
+ return parseHostsFrom(hosts);
+ }
+
+ private List parseHostsFrom(String hostConfig) {
+ List hostConfigs = Arrays.asList(hostConfig.split(";"));
+
+ return hostConfigs.stream()
+ .map(
+ hc -> {
+ String[] hostConfigValues = hc.split(":");
+ String host = hostConfigValues[0];
+ int port = Integer.parseInt(hostConfigValues[1]);
+ String rack = hostConfigValues[2];
+
+ if (hostConfigValues.length >= 4) {
+ String password = hostConfigValues[3];
+ return new HostBuilder()
+ .setHostname(host)
+ .setPort(port)
+ .setRack(rack)
+ .setStatus(Host.Status.Up)
+ .setPassword(password)
+ .createHost();
+ }
+ return new HostBuilder()
+ .setHostname(host)
+ .setPort(port)
+ .setRack(rack)
+ .setStatus(Host.Status.Up)
+ .createHost();
+ })
+ .collect(Collectors.toList());
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/LocalhostHostSupplier.java b/persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/LocalhostHostSupplier.java
new file mode 100644
index 0000000..7834271
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/dynoqueue/LocalhostHostSupplier.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2022 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.dynoqueue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.netflix.conductor.redis.config.RedisProperties;
+import com.netflix.dyno.connectionpool.Host;
+import com.netflix.dyno.connectionpool.HostBuilder;
+import com.netflix.dyno.connectionpool.HostSupplier;
+
+public class LocalhostHostSupplier implements HostSupplier {
+
+ private final RedisProperties properties;
+
+ public LocalhostHostSupplier(RedisProperties properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public List getHosts() {
+ Host dynoHost =
+ new HostBuilder()
+ .setHostname("localhost")
+ .setIpAddress("0")
+ .setRack(properties.getAvailabilityZone())
+ .setStatus(Host.Status.Up)
+ .createHost();
+ return Arrays.asList(dynoHost);
+ }
+}
diff --git a/persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisCluster.java b/persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisCluster.java
new file mode 100644
index 0000000..fc989bf
--- /dev/null
+++ b/persistence/src/main/java/com/netflix/conductor/redis/jedis/JedisCluster.java
@@ -0,0 +1,953 @@
+/*
+ * Copyright 2020 Orkes, Inc.
+ *
+ * Licensed under the Orkes Community License (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://github.com/orkes-io/licenses/blob/main/community/LICENSE.txt
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package com.netflix.conductor.redis.jedis;
+
+import java.util.AbstractMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import redis.clients.jedis.BitPosParams;
+import redis.clients.jedis.GeoCoordinate;
+import redis.clients.jedis.GeoRadiusResponse;
+import redis.clients.jedis.GeoUnit;
+import redis.clients.jedis.ListPosition;
+import redis.clients.jedis.ScanParams;
+import redis.clients.jedis.ScanResult;
+import redis.clients.jedis.SortingParams;
+import redis.clients.jedis.StreamConsumersInfo;
+import redis.clients.jedis.StreamEntry;
+import redis.clients.jedis.StreamEntryID;
+import redis.clients.jedis.StreamGroupInfo;
+import redis.clients.jedis.StreamInfo;
+import redis.clients.jedis.StreamPendingEntry;
+import redis.clients.jedis.Tuple;
+import redis.clients.jedis.commands.JedisCommands;
+import redis.clients.jedis.params.GeoRadiusParam;
+import redis.clients.jedis.params.SetParams;
+import redis.clients.jedis.params.ZAddParams;
+import redis.clients.jedis.params.ZIncrByParams;
+
+public class JedisCluster implements JedisCommands {
+
+ private final redis.clients.jedis.JedisCluster jedisCluster;
+
+ public JedisCluster(redis.clients.jedis.JedisCluster jedisCluster) {
+ this.jedisCluster = jedisCluster;
+ }
+
+ @Override
+ public String set(String key, String value) {
+ return jedisCluster.set(key, value);
+ }
+
+ @Override
+ public String set(String key, String value, SetParams params) {
+ return jedisCluster.set(key, value, params);
+ }
+
+ @Override
+ public String get(String key) {
+ return jedisCluster.get(key);
+ }
+
+ @Override
+ public Boolean exists(String key) {
+ return jedisCluster.exists(key);
+ }
+
+ @Override
+ public Long persist(String key) {
+ return jedisCluster.persist(key);
+ }
+
+ @Override
+ public String type(String key) {
+ return jedisCluster.type(key);
+ }
+
+ @Override
+ public byte[] dump(String key) {
+ return jedisCluster.dump(key);
+ }
+
+ @Override
+ public String restore(String key, int ttl, byte[] serializedValue) {
+ return jedisCluster.restore(key, ttl, serializedValue);
+ }
+
+ @Override
+ public String restoreReplace(String key, int ttl, byte[] serializedValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Long expire(String key, int seconds) {
+ return jedisCluster.expire(key, seconds);
+ }
+
+ @Override
+ public Long pexpire(String key, long milliseconds) {
+ return jedisCluster.pexpire(key, milliseconds);
+ }
+
+ @Override
+ public Long expireAt(String key, long unixTime) {
+ return jedisCluster.expireAt(key, unixTime);
+ }
+
+ @Override
+ public Long pexpireAt(String key, long millisecondsTimestamp) {
+ return jedisCluster.pexpireAt(key, millisecondsTimestamp);
+ }
+
+ @Override
+ public Long ttl(String key) {
+ return jedisCluster.ttl(key);
+ }
+
+ @Override
+ public Long pttl(String key) {
+ return jedisCluster.pttl(key);
+ }
+
+ @Override
+ public Long touch(String key) {
+ return jedisCluster.touch(key);
+ }
+
+ @Override
+ public Boolean setbit(String key, long offset, boolean value) {
+ return jedisCluster.setbit(key, offset, value);
+ }
+
+ @Override
+ public Boolean setbit(String key, long offset, String value) {
+ return jedisCluster.setbit(key, offset, value);
+ }
+
+ @Override
+ public Boolean getbit(String key, long offset) {
+ return jedisCluster.getbit(key, offset);
+ }
+
+ @Override
+ public Long setrange(String key, long offset, String value) {
+ return jedisCluster.setrange(key, offset, value);
+ }
+
+ @Override
+ public String getrange(String key, long startOffset, long endOffset) {
+ return jedisCluster.getrange(key, startOffset, endOffset);
+ }
+
+ @Override
+ public String getSet(String key, String value) {
+ return jedisCluster.getSet(key, value);
+ }
+
+ @Override
+ public Long setnx(String key, String value) {
+ return jedisCluster.setnx(key, value);
+ }
+
+ @Override
+ public String setex(String key, int seconds, String value) {
+ return jedisCluster.setex(key, seconds, value);
+ }
+
+ @Override
+ public String psetex(String key, long milliseconds, String value) {
+ return jedisCluster.psetex(key, milliseconds, value);
+ }
+
+ @Override
+ public Long decrBy(String key, long integer) {
+ return jedisCluster.decrBy(key, integer);
+ }
+
+ @Override
+ public Long decr(String key) {
+ return jedisCluster.decr(key);
+ }
+
+ @Override
+ public Long incrBy(String key, long integer) {
+ return jedisCluster.incrBy(key, integer);
+ }
+
+ @Override
+ public Double incrByFloat(String key, double value) {
+ return jedisCluster.incrByFloat(key, value);
+ }
+
+ @Override
+ public Long incr(String key) {
+ return jedisCluster.incr(key);
+ }
+
+ @Override
+ public Long append(String key, String value) {
+ return jedisCluster.append(key, value);
+ }
+
+ @Override
+ public String substr(String key, int start, int end) {
+ return jedisCluster.substr(key, start, end);
+ }
+
+ @Override
+ public Long hset(String key, String field, String value) {
+ return jedisCluster.hset(key, field, value);
+ }
+
+ @Override
+ public Long hset(String key, Map hash) {
+ return jedisCluster.hset(key, hash);
+ }
+
+ @Override
+ public String hget(String key, String field) {
+ return jedisCluster.hget(key, field);
+ }
+
+ @Override
+ public Long hsetnx(String key, String field, String value) {
+ return jedisCluster.hsetnx(key, field, value);
+ }
+
+ @Override
+ public String hmset(String key, Map hash) {
+ return jedisCluster.hmset(key, hash);
+ }
+
+ @Override
+ public List hmget(String key, String... fields) {
+ return jedisCluster.hmget(key, fields);
+ }
+
+ @Override
+ public Long hincrBy(String key, String field, long value) {
+ return jedisCluster.hincrBy(key, field, value);
+ }
+
+ @Override
+ public Double hincrByFloat(String key, String field, double value) {
+ return jedisCluster.hincrByFloat(key.getBytes(), field.getBytes(), value);
+ }
+
+ @Override
+ public Boolean hexists(String key, String field) {
+ return jedisCluster.hexists(key, field);
+ }
+
+ @Override
+ public Long hdel(String key, String... field) {
+ return jedisCluster.hdel(key, field);
+ }
+
+ @Override
+ public Long hlen(String key) {
+ return jedisCluster.hlen(key);
+ }
+
+ @Override
+ public Set hkeys(String key) {
+ return jedisCluster.hkeys(key);
+ }
+
+ @Override
+ public List hvals(String key) {
+ return jedisCluster.hvals(key);
+ }
+
+ @Override
+ public Map hgetAll(String key) {
+ return jedisCluster.hgetAll(key);
+ }
+
+ @Override
+ public Long rpush(String key, String... string) {
+ return jedisCluster.rpush(key, string);
+ }
+
+ @Override
+ public Long lpush(String key, String... string) {
+ return jedisCluster.lpush(key, string);
+ }
+
+ @Override
+ public Long llen(String key) {
+ return jedisCluster.llen(key);
+ }
+
+ @Override
+ public List lrange(String key, long start, long end) {
+ return jedisCluster.lrange(key, start, end);
+ }
+
+ @Override
+ public String ltrim(String key, long start, long end) {
+ return jedisCluster.ltrim(key, start, end);
+ }
+
+ @Override
+ public String lindex(String key, long index) {
+ return jedisCluster.lindex(key, index);
+ }
+
+ @Override
+ public String lset(String key, long index, String value) {
+ return jedisCluster.lset(key, index, value);
+ }
+
+ @Override
+ public Long lrem(String key, long count, String value) {
+ return jedisCluster.lrem(key, count, value);
+ }
+
+ @Override
+ public String lpop(String key) {
+ return jedisCluster.lpop(key);
+ }
+
+ @Override
+ public String rpop(String key) {
+ return jedisCluster.rpop(key);
+ }
+
+ @Override
+ public Long sadd(String key, String... member) {
+ return jedisCluster.sadd(key, member);
+ }
+
+ @Override
+ public Set smembers(String key) {
+ return jedisCluster.smembers(key);
+ }
+
+ @Override
+ public Long srem(String key, String... member) {
+ return jedisCluster.srem(key, member);
+ }
+
+ @Override
+ public String spop(String key) {
+ return jedisCluster.spop(key);
+ }
+
+ @Override
+ public Set spop(String key, long count) {
+ return jedisCluster.spop(key, count);
+ }
+
+ @Override
+ public Long scard(String key) {
+ return jedisCluster.scard(key);
+ }
+
+ @Override
+ public Boolean sismember(String key, String member) {
+ return jedisCluster.sismember(key, member);
+ }
+
+ @Override
+ public String srandmember(String key) {
+ return jedisCluster.srandmember(key);
+ }
+
+ @Override
+ public List srandmember(String key, int count) {
+ return jedisCluster.srandmember(key, count);
+ }
+
+ @Override
+ public Long strlen(String key) {
+ return jedisCluster.strlen(key);
+ }
+
+ @Override
+ public Long zadd(String key, double score, String member) {
+ return jedisCluster.zadd(key, score, member);
+ }
+
+ @Override
+ public Long zadd(String key, double score, String member, ZAddParams params) {
+ return jedisCluster.zadd(key, score, member, params);
+ }
+
+ @Override
+ public Long zadd(String key, Map scoreMembers) {
+ return jedisCluster.zadd(key, scoreMembers);
+ }
+
+ @Override
+ public Long zadd(String key, Map scoreMembers, ZAddParams params) {
+ return jedisCluster.zadd(key, scoreMembers, params);
+ }
+
+ @Override
+ public Set zrange(String key, long start, long end) {
+ return jedisCluster.zrange(key, start, end);
+ }
+
+ @Override
+ public Long zrem(String key, String... member) {
+ return jedisCluster.zrem(key, member);
+ }
+
+ @Override
+ public Double zincrby(String key, double score, String member) {
+ return jedisCluster.zincrby(key, score, member);
+ }
+
+ @Override
+ public Double zincrby(String key, double score, String member, ZIncrByParams params) {
+ return jedisCluster.zincrby(key, score, member, params);
+ }
+
+ @Override
+ public Long zrank(String key, String member) {
+ return jedisCluster.zrank(key, member);
+ }
+
+ @Override
+ public Long zrevrank(String key, String member) {
+ return jedisCluster.zrevrank(key, member);
+ }
+
+ @Override
+ public Set zrevrange(String key, long start, long end) {
+ return jedisCluster.zrevrange(key, start, end);
+ }
+
+ @Override
+ public Set zrangeWithScores(String key, long start, long end) {
+ return jedisCluster.zrangeWithScores(key, start, end);
+ }
+
+ @Override
+ public Set zrevrangeWithScores(String key, long start, long end) {
+ return jedisCluster.zrevrangeWithScores(key, start, end);
+ }
+
+ @Override
+ public Long zcard(String key) {
+ return jedisCluster.zcard(key);
+ }
+
+ @Override
+ public Double zscore(String key, String member) {
+ return jedisCluster.zscore(key, member);
+ }
+
+ @Override
+ public Tuple zpopmax(String key) {
+ return jedisCluster.zpopmax(key);
+ }
+
+ @Override
+ public Set zpopmax(String key, int count) {
+ return jedisCluster.zpopmax(key, count);
+ }
+
+ @Override
+ public Tuple zpopmin(String key) {
+ return jedisCluster.zpopmin(key);
+ }
+
+ @Override
+ public Set zpopmin(String key, int count) {
+ return jedisCluster.zpopmin(key, count);
+ }
+
+ @Override
+ public List sort(String key) {
+ return jedisCluster.sort(key);
+ }
+
+ @Override
+ public List sort(String key, SortingParams sortingParameters) {
+ return jedisCluster.sort(key, sortingParameters);
+ }
+
+ @Override
+ public Long zcount(String key, double min, double max) {
+ return jedisCluster.zcount(key, min, max);
+ }
+
+ @Override
+ public Long zcount(String key, String min, String max) {
+ return jedisCluster.zcount(key, min, max);
+ }
+
+ @Override
+ public Set zrangeByScore(String key, double min, double max) {
+ return jedisCluster.zrangeByScore(key, min, max);
+ }
+
+ @Override
+ public Set zrangeByScore(String key, String min, String max) {
+ return jedisCluster.zrangeByScore(key, min, max);
+ }
+
+ @Override
+ public Set zrevrangeByScore(String key, double max, double min) {
+ return jedisCluster.zrevrangeByScore(key, max, min);
+ }
+
+ @Override
+ public Set zrangeByScore(String key, double min, double max, int offset, int count) {
+ return jedisCluster.zrangeByScore(key, min, max, offset, count);
+ }
+
+ @Override
+ public Set zrevrangeByScore(String key, String max, String min) {
+ return jedisCluster.zrevrangeByScore(key, max, min);
+ }
+
+ @Override
+ public Set zrangeByScore(String key, String min, String max, int offset, int count) {
+ return jedisCluster.zrangeByScore(key, min, max, offset, count);
+ }
+
+ @Override
+ public Set zrevrangeByScore(String key, double max, double min, int offset, int count) {
+ return jedisCluster.zrevrangeByScore(key, max, min, offset, count);
+ }
+
+ @Override
+ public Set zrangeByScoreWithScores(String key, double min, double max) {
+ return jedisCluster.zrangeByScoreWithScores(key, min, max);
+ }
+
+ @Override
+ public Set zrevrangeByScoreWithScores(String key, double max, double min) {
+ return jedisCluster.zrevrangeByScoreWithScores(key, max, min);
+ }
+
+ @Override
+ public Set zrangeByScoreWithScores(
+ String key, double min, double max, int offset, int count) {
+ return jedisCluster.zrangeByScoreWithScores(key, min, max, offset, count);
+ }
+
+ @Override
+ public Set