Add multi Quartz scheduler functionality

cli/src/test/java/com/box/l10n/mojito/cli/command/ThirdPartySyncCommandTest.java

@@ -9,6 +9,7 @@
 import com.box.l10n.mojito.cli.utils.TestingJobListener;
 import com.box.l10n.mojito.entity.Repository;
 import com.box.l10n.mojito.json.ObjectMapper;
+import com.box.l10n.mojito.quartz.QuartzSchedulerManager;
 import com.box.l10n.mojito.rest.thirdparty.ThirdPartySyncAction;
 import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJob;
 import com.box.l10n.mojito.service.thirdparty.ThirdPartySyncJobInput;
@@ -19,7 +20,6 @@
 import org.junit.Test;
 import org.quartz.JobKey;
 import org.quartz.Matcher;
-import org.quartz.Scheduler;
 import org.quartz.SchedulerException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -30,7 +30,7 @@ public class ThirdPartySyncCommandTest extends CLITestBase {
   @Qualifier("fail_on_unknown_properties_false")
   ObjectMapper objectMapper;
 
-  @Autowired Scheduler scheduler;
+  @Autowired QuartzSchedulerManager schedulerManager;
 
   Matcher<JobKey> jobMatcher;
   TestingJobListener testingJobListener;
@@ -39,13 +39,20 @@ public class ThirdPartySyncCommandTest extends CLITestBase {
   public void setUp() throws SchedulerException {
     testingJobListener = new TestingJobListener(objectMapper);
     jobMatcher = new PollableTaskJobMatcher<>(ThirdPartySyncJob.class);
-    scheduler.getListenerManager().addJobListener(testingJobListener, jobMatcher);
+    schedulerManager
+        .getScheduler(QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME)
+        .getListenerManager()
+        .addJobListener(testingJobListener, jobMatcher);
   }
 
   @After
   public void tearDown() throws SchedulerException {
-    scheduler.getListenerManager().removeJobListener(testingJobListener.getName());
-    scheduler
+    schedulerManager
+        .getScheduler(QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME)
+        .getListenerManager()
+        .removeJobListener(testingJobListener.getName());
+    schedulerManager
+        .getScheduler(QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME)
         .getListenerManager()
         .removeJobListenerMatcher(testingJobListener.getName(), jobMatcher);
   }

docker/docker-compose-api-worker.yml

@@ -19,10 +19,10 @@ services:
       - --collation-server=utf8mb4_bin
       - --max-connections=1000
       - --log_error_verbosity=2
-  worker:
+  api:
     deploy:
       mode: replicated
-      replicas: 2
+      replicas: 1
       endpoint_mode: vip
       resources:
         limits:
@@ -39,45 +39,65 @@ services:
     depends_on:
       db:
         condition: service_healthy
-      api:
-        condition: service_healthy
     build:
       dockerfile: docker/Dockerfile-bk8
       context: ../
+    image: mojito:latest
+    pull_policy: never
     links:
       - db
+    ports:
+      - "8080:8080"
     restart: always
     environment:
       SPRING_APPLICATION_JSON: '{
-        "spring.flyway.enabled": "true",
-        "l10n.flyway.clean" : "false",
-        "spring.jpa.database-platform" : "org.hibernate.dialect.MySQLDialect",
-        "spring.jpa.hibernate.ddl-auto" : "none",
-        "spring.datasource.url" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
-        "spring.datasource.username" : "mojito",
-        "spring.datasource.password" : "ChangeMe",
-        "spring.datasource.driverClassName" : "com.mysql.cj.jdbc.Driver",
-        "spring.jpa.defer-datasource-initialization" : "false",
-        "l10n.org.quartz.scheduler.enabled" : "true",
-        "l10n.org.quartz.jobStore.useProperties" : "true",
-        "l10n.org.quartz.scheduler.instanceId" : "AUTO",
-        "l10n.org.quartz.jobStore.isClustered" : "true",
-        "l10n.org.quartz.threadPool.threadCount" : "10",
-        "l10n.org.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX",
-        "l10n.org.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate",
-        "l10n.org.quartz.jobStore.dataSource" : "myDS",
-        "l10n.org.quartz.dataSource.myDS.provider" : "hikaricp",
-        "l10n.org.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver",
-        "l10n.org.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
-        "l10n.org.quartz.dataSource.myDS.user" : "mojito",
-        "l10n.org.quartz.dataSource.myDS.password" : "ChangeMe",
-        "l10n.org.quartz.dataSource.myDS.maxConnections" : "12",
-        "l10n.org.quartz.dataSource.myDS.validationQuery" : "select 1"
-        }'
-  api:
+      "spring.flyway.enabled": "true",
+      "l10n.flyway.clean" : "false",
+      "spring.jpa.database-platform" : "org.hibernate.dialect.MySQLDialect",
+      "spring.jpa.hibernate.ddl-auto" : "none",
+      "spring.datasource.url" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
+      "spring.datasource.username" : "mojito",
+      "spring.datasource.password" : "ChangeMe",
+      "spring.datasource.driverClassName" : "com.mysql.cj.jdbc.Driver",
+      "spring.jpa.defer-datasource-initialization" : "false",
+      "l10n.org.quartz.scheduler.enabled" : "false",
+      "l10n.org.multi-quartz.enabled" : "true",
+      "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.useProperties" : "true",
+      "l10n.org.multi-quartz.schedulers.default.quartz.scheduler.instanceId" : "AUTO",
+      "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.isClustered" : "true",
+      "l10n.org.multi-quartz.schedulers.default.quartz.threadPool.threadCount" : 10,
+      "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX",
+      "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate",
+      "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.dataSource" : "myDS",
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.provider" : "hikaricp",
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver",
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.user" : "mojito",
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.password" : "ChangeMe",
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.maxConnections" : 12,
+      "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.validationQuery" : "select 1",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.useProperties" : "true",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.scheduler.instanceId" : "AUTO",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.isClustered" : "true",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.threadPool.threadCount" : 5,
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.dataSource" : "myDS",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.provider" : "hikaricp",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.user" : "mojito",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.password" : "ChangeMe",
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.maxConnections" : 12,
+      "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.validationQuery" : "select 1",
+      "l10n.assetExtraction.quartz.schedulerName" : "lowPriority",
+      "logging.level.com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler" : "DEBUG",
+      "logging.level.com.box.l10n.mojito.service.repository.statistics.RepositoryStatisticsJob" : "DEBUG"
+      }'
+  worker:
     deploy:
       mode: replicated
-      replicas: 1
+      replicas: 2
       endpoint_mode: vip
       resources:
         limits:
@@ -94,38 +114,55 @@ services:
     depends_on:
       db:
         condition: service_healthy
-    build:
-      dockerfile: docker/Dockerfile-bk8
-      context: ../
+      api:
+        condition: service_healthy
+    image: mojito:latest
+    pull_policy: never
     links:
       - db
-    ports:
-      - "8080:8080"
     restart: always
     environment:
       SPRING_APPLICATION_JSON: '{
-      "spring.flyway.enabled": "true",
-      "l10n.flyway.clean" : "false",
-      "spring.jpa.database-platform" : "org.hibernate.dialect.MySQLDialect",
-      "spring.jpa.hibernate.ddl-auto" : "none",
-      "spring.datasource.url" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
-      "spring.datasource.username" : "mojito",
-      "spring.datasource.password" : "ChangeMe",
-      "spring.datasource.driverClassName" : "com.mysql.cj.jdbc.Driver",
-      "spring.jpa.defer-datasource-initialization" : "false",
-      "l10n.org.quartz.scheduler.enabled" : "false",
-      "l10n.org.quartz.jobStore.useProperties" : "true",
-      "l10n.org.quartz.scheduler.instanceId" : "AUTO",
-      "l10n.org.quartz.jobStore.isClustered" : "true",
-      "l10n.org.quartz.threadPool.threadCount" : "10",
-      "l10n.org.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX",
-      "l10n.org.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate",
-      "l10n.org.quartz.jobStore.dataSource" : "myDS",
-      "l10n.org.quartz.dataSource.myDS.provider" : "hikaricp",
-      "l10n.org.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver",
-      "l10n.org.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
-      "l10n.org.quartz.dataSource.myDS.user" : "mojito",
-      "l10n.org.quartz.dataSource.myDS.password" : "ChangeMe",
-      "l10n.org.quartz.dataSource.myDS.maxConnections" : "12",
-      "l10n.org.quartz.dataSource.myDS.validationQuery" : "select 1"
-      }'
+        "spring.flyway.enabled": "true",
+        "l10n.flyway.clean" : "false",
+        "spring.jpa.database-platform" : "org.hibernate.dialect.MySQLDialect",
+        "spring.jpa.hibernate.ddl-auto" : "none",
+        "spring.datasource.url" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
+        "spring.datasource.username" : "mojito",
+        "spring.datasource.password" : "ChangeMe",
+        "spring.datasource.driverClassName" : "com.mysql.cj.jdbc.Driver",
+        "spring.jpa.defer-datasource-initialization" : "false",
+        "l10n.org.quartz.scheduler.enabled" : "true",
+        "l10n.org.multi-quartz.enabled" : "true",
+        "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.useProperties" : "true",
+        "l10n.org.multi-quartz.schedulers.default.quartz.scheduler.instanceId" : "AUTO",
+        "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.isClustered" : "true",
+        "l10n.org.multi-quartz.schedulers.default.quartz.threadPool.threadCount" : 10,
+        "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX",
+        "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate",
+        "l10n.org.multi-quartz.schedulers.default.quartz.jobStore.dataSource" : "myDS",
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.provider" : "hikaricp",
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver",
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.user" : "mojito",
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.password" : "ChangeMe",
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.maxConnections" : 12,
+        "l10n.org.multi-quartz.schedulers.default.quartz.dataSource.myDS.validationQuery" : "select 1",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.useProperties" : "true",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.scheduler.instanceId" : "AUTO",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.isClustered" : "true",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.threadPool.threadCount" : 5,
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.jobStore.dataSource" : "myDS",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.provider" : "hikaricp",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.user" : "mojito",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.password" : "ChangeMe",
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.maxConnections" : 12,
+        "l10n.org.multi-quartz.schedulers.lowPriority.quartz.dataSource.myDS.validationQuery" : "select 1",
+        "l10n.assetExtraction.quartz.schedulerName" : "lowPriority",
+        "logging.level.com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler" : "DEBUG",
+        "logging.level.com.box.l10n.mojito.service.repository.statistics.RepositoryStatisticsJob" : "DEBUG"
+        }'

pom.xml

@@ -36,6 +36,8 @@
         <jjwt.version>0.10.5</jjwt.version>
         <github.api.version>1.313</github.api.version>
         <icu4j.version>64.2</icu4j.version>
+        <docker.compose.detached.mode>true</docker.compose.detached.mode>
+        <docker.compose.remove.volumes>true</docker.compose.remove.volumes>
     </properties>
 
     <dependencies>
@@ -150,6 +152,34 @@
                     </dependency>
                 </dependencies>
             </plugin>
+            <plugin>
+                <groupId>com.dkanejs.maven.plugins</groupId>
+                <artifactId>docker-compose-maven-plugin</artifactId>
+                <version>4.0.0</version>
+                <configuration>
+                    <composeFiles>
+                        <composeFile>./docker/docker-compose-api-worker.yml</composeFile>
+                    </composeFiles>
+                    <ignorePullFailures>true</ignorePullFailures>
+                    <detachedMode>${docker.compose.detached.mode}</detachedMode>
+                    <removeVolumes>${docker.compose.remove.volumes}</removeVolumes>
+                    <build>true</build>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>api-worker-cluster-up</id>
+                        <goals>
+                            <goal>up</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>api-worker-cluster-down</id>
+                        <goals>
+                            <goal>down</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java

@@ -1,9 +1,10 @@
 package com.box.l10n.mojito.quartz;
 
+import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Properties;
 import java.util.Set;
 import javax.annotation.PostConstruct;
 import org.quartz.JobDetail;
@@ -16,6 +17,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 @Component
@@ -25,37 +27,47 @@ public class QuartzConfig {
 
   public static final String DYNAMIC_GROUP_NAME = "DYNAMIC";
 
-  @Autowired Scheduler scheduler;
+  @Autowired QuartzSchedulerManager schedulerManager;
 
   @Autowired(required = false)
   List<Trigger> triggers = new ArrayList<>();
 
   @Autowired(required = false)
   List<JobDetail> jobDetails = new ArrayList<>();
 
-  @Autowired QuartzPropertiesConfig quartzPropertiesConfig;
+  @Value("${l10n.org.quartz.scheduler.enabled:true}")
+  Boolean schedulerEnabled;
 
   /**
    * Starts the scheduler after having removed outdated trigger/jobs
    *
    * @throws SchedulerException
    */
   @PostConstruct
-  void startScheduler() throws SchedulerException {
-    Properties quartzProps = quartzPropertiesConfig.getQuartzProperties();
+  void startSchedulers() throws SchedulerException {
     removeOutdatedJobs();
-    if (Boolean.parseBoolean(quartzProps.getProperty("org.quartz.scheduler.enabled", "true"))) {
-      logger.info("Starting scheduler");
-      scheduler.startDelayed(2);
+    int delay = 2;
+    if (schedulerEnabled) {
+      for (Scheduler scheduler : schedulerManager.getSchedulers()) {
+        logger.info("Starting scheduler: {}", scheduler.getSchedulerName());
+        scheduler.startDelayed(delay);
+        // Increment the delay to avoid lock exceptions being thrown as both schedulers try to start
+        // concurrently
+        delay++;
+      }
     }
   }
 
   void removeOutdatedJobs() throws SchedulerException {
-    scheduler.unscheduleJobs(new ArrayList<TriggerKey>(getOutdatedTriggerKeys()));
-    scheduler.deleteJobs(new ArrayList<JobKey>(getOutdatedJobKeys()));
+    for (Scheduler scheduler : schedulerManager.getSchedulers()) {
+      if (scheduler.getSchedulerName().equals(DEFAULT_SCHEDULER_NAME)) {
+        scheduler.unscheduleJobs(new ArrayList<TriggerKey>(getOutdatedTriggerKeys(scheduler)));
+      }
+      scheduler.deleteJobs(new ArrayList<JobKey>(getOutdatedJobKeys(scheduler)));
+    }
   }
 
-  Set<JobKey> getOutdatedJobKeys() throws SchedulerException {
+  Set<JobKey> getOutdatedJobKeys(Scheduler scheduler) throws SchedulerException {
 
     Set<JobKey> jobKeys =
         scheduler.getJobKeys(GroupMatcher.jobGroupEquals(Scheduler.DEFAULT_GROUP));
@@ -72,7 +84,7 @@ Set<JobKey> getOutdatedJobKeys() throws SchedulerException {
     return jobKeys;
   }
 
-  Set<TriggerKey> getOutdatedTriggerKeys() throws SchedulerException {
+  Set<TriggerKey> getOutdatedTriggerKeys(Scheduler scheduler) throws SchedulerException {
 
     Set<TriggerKey> triggerKeys =
         scheduler.getTriggerKeys(GroupMatcher.triggerGroupEquals(Scheduler.DEFAULT_GROUP));