From e4be83e63a05d5002ee5a0a434cbf69d1aa9e8c2 Mon Sep 17 00:00:00 2001 From: Mark Allen <3417310+maallen@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:21:47 +0100 Subject: [PATCH] Initial implementation with properties configured schedulers --- docker/docker-compose-api-worker.yml | 92 +++++++++++++------ .../box/l10n/mojito/quartz/QuartzConfig.java | 27 ++++-- .../quartz/QuartzPollableTaskScheduler.java | 14 +-- .../mojito/quartz/QuartzPropertiesConfig.java | 8 ++ .../box/l10n/mojito/quartz/QuartzQueue.java | 17 ---- .../mojito/quartz/QuartzQueueException.java | 12 +++ .../mojito/quartz/QuartzSchedulerConfig.java | 71 ++++---------- .../mojito/quartz/QuartzSchedulerManager.java | 31 +------ .../box/l10n/mojito/quartz/QuartzService.java | 14 +-- .../quartz/QuartzSingleSchedulerManager.java | 28 ++++++ .../multi/QuartzMultiSchedulerConfig.java | 22 +++++ .../multi/QuartzMultiSchedulerException.java | 12 +++ .../multi/QuartzMultiSchedulerFactory.java | 88 ++++++++++++++++++ .../multi/QuartzMultiSchedulerManager.java | 63 +++++++++++++ .../mojito/quartz/multi/SchedulerConfig.java | 35 +++++++ .../AssetExtractionService.java | 7 +- 16 files changed, 392 insertions(+), 149 deletions(-) delete mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueueException.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSingleSchedulerManager.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerConfig.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerException.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerFactory.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerManager.java create mode 100644 webapp/src/main/java/com/box/l10n/mojito/quartz/multi/SchedulerConfig.java diff --git a/docker/docker-compose-api-worker.yml b/docker/docker-compose-api-worker.yml index c6193514a6..486688e3a4 100644 --- a/docker/docker-compose-api-worker.yml +++ b/docker/docker-compose-api-worker.yml @@ -59,20 +59,38 @@ services: "spring.datasource.driverClassName" : "com.mysql.cj.jdbc.Driver", "spring.jpa.defer-datasource-initialization" : "false", "l10n.org.quartz.scheduler.enabled" : "true", - "l10n.org.quartz.jobStore.useProperties" : "true", - "l10n.org.quartz.scheduler.instanceId" : "AUTO", - "l10n.org.quartz.jobStore.isClustered" : "true", - "l10n.org.quartz.threadPool.threadCount" : "10", - "l10n.org.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX", - "l10n.org.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate", - "l10n.org.quartz.jobStore.dataSource" : "myDS", - "l10n.org.quartz.dataSource.myDS.provider" : "hikaricp", - "l10n.org.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver", - "l10n.org.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true", - "l10n.org.quartz.dataSource.myDS.user" : "mojito", - "l10n.org.quartz.dataSource.myDS.password" : "ChangeMe", - "l10n.org.quartz.dataSource.myDS.maxConnections" : "12", - "l10n.org.quartz.dataSource.myDS.validationQuery" : "select 1" + "l10n.org.multi-quartz.enabled" : "true", + "l10n.org.multi-quartz.schedulerConfigs[0].name" : "defaultScheduler", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.useProperties" : "true", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.scheduler.instanceId" : "AUTO", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.isClustered" : "true", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.threadPool.threadCount" : 10, + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.dataSource" : "myDS", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.provider" : "hikaricp", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.user" : "mojito", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.password" : "ChangeMe", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.maxConnections" : 12, + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.validationQuery" : "select 1", + "l10n.org.multi-quartz.schedulerConfigs[1].name" : "lowPriority", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.useProperties" : "true", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.scheduler.instanceId" : "AUTO", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.isClustered" : "true", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.threadPool.threadCount" : 5, + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.dataSource" : "myDS", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.provider" : "hikaricp", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.user" : "mojito", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.password" : "ChangeMe", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.maxConnections" : 12, + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.validationQuery" : "select 1", + "l10n.assetExtraction.quartz.schedulerName" : "lowPriority" }' api: deploy: @@ -114,18 +132,36 @@ services: "spring.datasource.driverClassName" : "com.mysql.cj.jdbc.Driver", "spring.jpa.defer-datasource-initialization" : "false", "l10n.org.quartz.scheduler.enabled" : "false", - "l10n.org.quartz.jobStore.useProperties" : "true", - "l10n.org.quartz.scheduler.instanceId" : "AUTO", - "l10n.org.quartz.jobStore.isClustered" : "true", - "l10n.org.quartz.threadPool.threadCount" : "10", - "l10n.org.quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX", - "l10n.org.quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate", - "l10n.org.quartz.jobStore.dataSource" : "myDS", - "l10n.org.quartz.dataSource.myDS.provider" : "hikaricp", - "l10n.org.quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver", - "l10n.org.quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true", - "l10n.org.quartz.dataSource.myDS.user" : "mojito", - "l10n.org.quartz.dataSource.myDS.password" : "ChangeMe", - "l10n.org.quartz.dataSource.myDS.maxConnections" : "12", - "l10n.org.quartz.dataSource.myDS.validationQuery" : "select 1" + "l10n.org.multi-quartz.enabled" : "true", + "l10n.org.multi-quartz.schedulerConfigs[0].name" : "defaultScheduler", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.useProperties" : "true", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.scheduler.instanceId" : "AUTO", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.isClustered" : "true", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.threadPool.threadCount" : 10, + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.jobStore.dataSource" : "myDS", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.provider" : "hikaricp", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.user" : "mojito", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.password" : "ChangeMe", + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.maxConnections" : 12, + "l10n.org.multi-quartz.schedulerConfigs[0].quartz.dataSource.myDS.validationQuery" : "select 1", + "l10n.org.multi-quartz.schedulerConfigs[1].name" : "lowPriority", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.useProperties" : "true", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.scheduler.instanceId" : "AUTO", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.isClustered" : "true", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.threadPool.threadCount" : 5, + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.class" : "org.quartz.impl.jdbcjobstore.JobStoreTX", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.driverDelegateClass" : "org.quartz.impl.jdbcjobstore.StdJDBCDelegate", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.jobStore.dataSource" : "myDS", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.provider" : "hikaricp", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.driver" : "com.mysql.jdbc.Driver", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.URL" : "jdbc:mysql://db:3306/mojito?characterEncoding=UTF-8&useUnicode=true", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.user" : "mojito", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.password" : "ChangeMe", + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.maxConnections" : 12, + "l10n.org.multi-quartz.schedulerConfigs[1].quartz.dataSource.myDS.validationQuery" : "select 1", + "l10n.assetExtraction.quartz.schedulerName" : "lowPriority" }' \ No newline at end of file diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java index 4f99d5ac80..0a1d7a678e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzConfig.java @@ -1,9 +1,10 @@ package com.box.l10n.mojito.quartz; +import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Properties; import java.util.Set; import javax.annotation.PostConstruct; import org.quartz.JobDetail; @@ -16,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @Component @@ -25,7 +27,7 @@ public class QuartzConfig { public static final String DYNAMIC_GROUP_NAME = "DYNAMIC"; - @Autowired List schedulers; + @Autowired QuartzSchedulerManager schedulerManager; @Autowired(required = false) List triggers = new ArrayList<>(); @@ -33,7 +35,8 @@ public class QuartzConfig { @Autowired(required = false) List jobDetails = new ArrayList<>(); - @Autowired QuartzPropertiesConfig quartzPropertiesConfig; + @Value("${l10n.org.quartz.scheduler.enabled:true}") + Boolean schedulerEnabled; /** * Starts the scheduler after having removed outdated trigger/jobs @@ -42,24 +45,28 @@ public class QuartzConfig { */ @PostConstruct void startSchedulers() throws SchedulerException { - Properties quartzProps = quartzPropertiesConfig.getQuartzProperties(); removeOutdatedJobs(); - if (Boolean.parseBoolean(quartzProps.getProperty("org.quartz.scheduler.enabled", "true"))) { - logger.info("Starting schedulers"); - for (Scheduler scheduler : schedulers) { - scheduler.startDelayed(2); + int delay = 2; + if (schedulerEnabled) { + for (Scheduler scheduler : schedulerManager.getSchedulers()) { + logger.info("Starting scheduler: {}", scheduler.getSchedulerName()); + scheduler.startDelayed(delay); + delay++; } } } void removeOutdatedJobs() throws SchedulerException { - for (Scheduler scheduler : schedulers) { - scheduler.unscheduleJobs(new ArrayList(getOutdatedTriggerKeys(scheduler))); + for (Scheduler scheduler : schedulerManager.getSchedulers()) { + if (scheduler.getSchedulerName().equals(DEFAULT_SCHEDULER_NAME)) { + scheduler.unscheduleJobs(new ArrayList(getOutdatedTriggerKeys(scheduler))); + } scheduler.deleteJobs(new ArrayList(getOutdatedJobKeys(scheduler))); } } Set getOutdatedJobKeys(Scheduler scheduler) throws SchedulerException { + Set jobKeys = scheduler.getJobKeys(GroupMatcher.jobGroupEquals(Scheduler.DEFAULT_GROUP)); Set newJobKeys = new HashSet<>(); diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java index d3f26dab3b..d7a3c624e8 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPollableTaskScheduler.java @@ -1,7 +1,7 @@ package com.box.l10n.mojito.quartz; import static com.box.l10n.mojito.quartz.QuartzConfig.DYNAMIC_GROUP_NAME; -import static com.box.l10n.mojito.quartz.QuartzQueue.DEFAULT; +import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; import com.box.l10n.mojito.entity.PollableTask; import com.box.l10n.mojito.json.ObjectMapper; @@ -42,7 +42,7 @@ public PollableFuture scheduleJob( Class> clazz, I input) { QuartzJobInfo quartzJobInfo = QuartzJobInfo.newBuilder(clazz).withInput(input).withMessage(clazz.getSimpleName()).build(); - return scheduleJob(quartzJobInfo, DEFAULT); + return scheduleJob(quartzJobInfo, DEFAULT_SCHEDULER_NAME); } public PollableFuture scheduleJobWithCustomTimeout( @@ -54,11 +54,11 @@ public PollableFuture scheduleJobWithCustomTimeout( .withMessage(clazz.getSimpleName()) .build(); - return scheduleJob(quartzJobInfo, DEFAULT); + return scheduleJob(quartzJobInfo, DEFAULT_SCHEDULER_NAME); } public PollableFuture scheduleJob(QuartzJobInfo quartzJobInfo) { - return scheduleJob(quartzJobInfo, DEFAULT); + return scheduleJob(quartzJobInfo, DEFAULT_SCHEDULER_NAME); } /** @@ -80,11 +80,11 @@ public PollableFuture scheduleJob(QuartzJobInfo quartzJobInfo) { * @return */ public PollableFuture scheduleJob( - QuartzJobInfo quartzJobInfo, QuartzQueue quartzQueue) { + QuartzJobInfo quartzJobInfo, String schedulerName) { - Scheduler scheduler = schedulerManager.getScheduler(quartzQueue); + Scheduler scheduler = schedulerManager.getScheduler(schedulerName); - logger.debug("Scheduling job on queue: {}", quartzQueue.getDescription()); + logger.debug("Scheduling job on queue: {}", schedulerName); String pollableTaskName = getPollableTaskName(quartzJobInfo.getClazz()); diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPropertiesConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPropertiesConfig.java index 6eebad22c3..d0bd17714a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPropertiesConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzPropertiesConfig.java @@ -5,12 +5,17 @@ import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration @ConfigurationProperties("l10n.org") +@ConditionalOnProperty( + name = "l10n.org.multi-quartz.enabled", + havingValue = "false", + matchIfMissing = true) public class QuartzPropertiesConfig { static Logger logger = LoggerFactory.getLogger(QuartzPropertiesConfig.class); @@ -31,6 +36,9 @@ public Properties getQuartzProperties() { logger.debug("org.quartz.{}={}", entry.getKey(), entry.getValue()); } + properties.put( + "org.quartz.scheduler.instanceName", QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME); + return properties; } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java deleted file mode 100644 index 00c8d2ee01..0000000000 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueue.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.box.l10n.mojito.quartz; - -public enum QuartzQueue { - LOW_PRIORITY("Low priority"), - DEFAULT("Default"), - HIGH_PRIORITY("High priority"); - - private String description; - - QuartzQueue(String description) { - this.description = description; - } - - public String getDescription() { - return description; - } -} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueueException.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueueException.java new file mode 100644 index 0000000000..eb8ab4cd6d --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzQueueException.java @@ -0,0 +1,12 @@ +package com.box.l10n.mojito.quartz; + +public class QuartzQueueException extends RuntimeException { + + public QuartzQueueException(String message) { + super(message); + } + + public QuartzQueueException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java index 3aa569d29c..21ef56ddd1 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerConfig.java @@ -5,11 +5,12 @@ import java.util.List; import java.util.Properties; import javax.sql.DataSource; +import org.quartz.SchedulerException; import org.quartz.Trigger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -18,6 +19,10 @@ import org.springframework.transaction.PlatformTransactionManager; @Configuration +@ConditionalOnProperty( + name = "l10n.org.multi-quartz.enabled", + havingValue = "false", + matchIfMissing = true) public class QuartzSchedulerConfig { /** logger */ @@ -50,52 +55,24 @@ public class QuartzSchedulerConfig { * removed. * * @return + * @throws SchedulerException */ + @Bean + public SchedulerFactoryBean scheduler() throws SchedulerException { - // TODO (maallen): Update config to read multiple schedulers config from app.properties so each - // scheduler can have it's own configured thread pool. - @Bean(name = "defaultScheduler") - public SchedulerFactoryBean defaultScheduler( - @Qualifier("defaultJobFactory") SpringBeanJobFactory jobFactory) { + logger.info("Create SchedulerFactoryBean"); - logger.info("Create default Scheduler"); - - Properties quartzProperties = quartzPropertiesConfig.getQuartzProperties(); - quartzProperties.put("org.quartz.scheduler.instanceName", "defaultScheduler"); - - SchedulerFactoryBean factory = getSchedulerFactory(quartzProperties, jobFactory); - factory.setTriggers(triggers.toArray(new Trigger[] {})); - return factory; - } - - @Bean(name = "lowPriorityScheduler") - public SchedulerFactoryBean lowPriorityScheduler( - @Qualifier("lowPriorityJobFactory") SpringBeanJobFactory jobFactory) { - - logger.info("Create Low Priority Scheduler"); - - Properties quartzProperties = quartzPropertiesConfig.getQuartzProperties(); - quartzProperties.put("org.quartz.scheduler.instanceName", "lowPriorityScheduler"); - - return getSchedulerFactory(quartzProperties, jobFactory); - } - - @Bean(name = "highPriorityScheduler") - public SchedulerFactoryBean highPriorityScheduler( - @Qualifier("highPriorityJobFactory") SpringBeanJobFactory jobFactory) { - logger.info("Create High Priority Scheduler"); Properties quartzProperties = quartzPropertiesConfig.getQuartzProperties(); - quartzProperties.put("org.quartz.scheduler.instanceName", "highPriorityScheduler"); - return getSchedulerFactory(quartzProperties, jobFactory); - } - private SchedulerFactoryBean getSchedulerFactory( - Properties quartzProperties, SpringBeanJobFactory springBeanJobFactory) { SchedulerFactoryBean schedulerFactory = new SchedulerFactoryBean(); + + String dataSource = quartzProperties.getProperty("org.quartz.jobStore.dataSource"); schedulerFactory.setQuartzProperties(quartzProperties); - schedulerFactory.setJobFactory(springBeanJobFactory); + schedulerFactory.setJobFactory(springBeanJobFactory()); schedulerFactory.setOverwriteExistingJobs(true); + schedulerFactory.setTriggers(triggers.toArray(new Trigger[] {})); schedulerFactory.setAutoStartup(false); + schedulerFactory.setBeanName(QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME); if (quartzMetricsReportingJobListener != null) { schedulerFactory.setGlobalJobListeners(quartzMetricsReportingJobListener); @@ -104,22 +81,8 @@ private SchedulerFactoryBean getSchedulerFactory( return schedulerFactory; } - @Bean(name = "lowPriorityJobFactory") - public SpringBeanJobFactory lowPrioritySpringBeanJobFactory() { - AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); - jobFactory.setApplicationContext(applicationContext); - return jobFactory; - } - - @Bean(name = "defaultJobFactory") - public SpringBeanJobFactory defaultSpringBeanJobFactory() { - AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); - jobFactory.setApplicationContext(applicationContext); - return jobFactory; - } - - @Bean(name = "highPriorityJobFactory") - public SpringBeanJobFactory highPrioritySpringBeanJobFactory() { + @Bean + public SpringBeanJobFactory springBeanJobFactory() { AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); jobFactory.setApplicationContext(applicationContext); return jobFactory; diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java index 0508d4dffa..6f11701bec 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSchedulerManager.java @@ -1,34 +1,13 @@ package com.box.l10n.mojito.quartz; +import java.util.List; import org.quartz.Scheduler; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -@Component -public class QuartzSchedulerManager { +public interface QuartzSchedulerManager { - @Autowired - @Qualifier("lowPriorityScheduler") - Scheduler lowPriorityScheduler; + String DEFAULT_SCHEDULER_NAME = "defaultScheduler"; - @Autowired - @Qualifier("defaultScheduler") - Scheduler defaultScheduler; + Scheduler getScheduler(String schedulerName); - @Autowired - @Qualifier("highPriorityScheduler") - Scheduler highPriorityScheduler; - - public Scheduler getScheduler(QuartzQueue quartzQueue) { - switch (quartzQueue) { - case LOW_PRIORITY: - return lowPriorityScheduler; - case HIGH_PRIORITY: - return highPriorityScheduler; - case DEFAULT: - default: - return defaultScheduler; - } - } + List getSchedulers(); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java index b961f77383..05cbe04f8a 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzService.java @@ -13,7 +13,6 @@ import org.quartz.impl.matchers.GroupMatcher; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @Service @@ -22,17 +21,20 @@ public class QuartzService { /** logger */ static Logger logger = getLogger(QuartzService.class); - @Autowired - @Qualifier("defaultScheduler") - Scheduler scheduler; + @Autowired QuartzSchedulerManager schedulerManager; - // TODO(mallen): Add handling for an injected list of all Schedulers instead. + // TODO(mallen): Handle for other schedulers other than 'default'??? public List getDynamicJobs() throws SchedulerException { - Set jobKeys = scheduler.getJobKeys(GroupMatcher.jobGroupEquals(DYNAMIC_GROUP_NAME)); + Set jobKeys = + schedulerManager + .getScheduler(QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME) + .getJobKeys(GroupMatcher.jobGroupEquals(DYNAMIC_GROUP_NAME)); return jobKeys.stream().map(jobKey -> jobKey.getName()).collect(Collectors.toList()); } public void deleteAllDynamicJobs() throws SchedulerException { + Scheduler scheduler = + schedulerManager.getScheduler(QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME); Set jobKeys = scheduler.getJobKeys(GroupMatcher.jobGroupEquals(DYNAMIC_GROUP_NAME)); scheduler.deleteJobs(new ArrayList<>(jobKeys)); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSingleSchedulerManager.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSingleSchedulerManager.java new file mode 100644 index 0000000000..9cbf61ce85 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/QuartzSingleSchedulerManager.java @@ -0,0 +1,28 @@ +package com.box.l10n.mojito.quartz; + +import java.util.Collections; +import java.util.List; +import org.quartz.Scheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty( + name = "l10n.org.multi-quartz.enabled", + havingValue = "false", + matchIfMissing = true) +public class QuartzSingleSchedulerManager implements QuartzSchedulerManager { + + @Autowired Scheduler scheduler; + + @Override + public Scheduler getScheduler(String schedulerName) { + return scheduler; + } + + @Override + public List getSchedulers() { + return Collections.singletonList(scheduler); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerConfig.java new file mode 100644 index 0000000000..40f21fa5a1 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerConfig.java @@ -0,0 +1,22 @@ +package com.box.l10n.mojito.quartz.multi; + +import java.util.List; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties("l10n.org.multi-quartz") +@ConditionalOnProperty(name = "l10n.org.multi-quartz.enabled", havingValue = "true") +public class QuartzMultiSchedulerConfig { + + List schedulerConfigs; + + public List getSchedulerConfigs() { + return schedulerConfigs; + } + + public void setSchedulerConfigs(List schedulerConfigs) { + this.schedulerConfigs = schedulerConfigs; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerException.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerException.java new file mode 100644 index 0000000000..41bc7c660c --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerException.java @@ -0,0 +1,12 @@ +package com.box.l10n.mojito.quartz.multi; + +public class QuartzMultiSchedulerException extends RuntimeException { + + public QuartzMultiSchedulerException(String message) { + super(message); + } + + public QuartzMultiSchedulerException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerFactory.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerFactory.java new file mode 100644 index 0000000000..20825986a2 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerFactory.java @@ -0,0 +1,88 @@ +package com.box.l10n.mojito.quartz.multi; + +import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; + +import com.box.l10n.mojito.monitoring.QuartzMetricsReportingJobListener; +import com.box.l10n.mojito.quartz.AutoWiringSpringBeanJobFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import javax.annotation.PostConstruct; +import org.quartz.Trigger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; +import org.springframework.scheduling.quartz.SpringBeanJobFactory; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(name = "l10n.org.multi-quartz.enabled", havingValue = "true") +public class QuartzMultiSchedulerFactory { + + Logger logger = LoggerFactory.getLogger(QuartzMultiSchedulerFactory.class); + + @Autowired ApplicationContext applicationContext; + + @Autowired QuartzMultiSchedulerConfig quartzMultiSchedulerConfig; + + @Autowired(required = false) + QuartzMetricsReportingJobListener quartzMetricsReportingJobListener; + + @Autowired(required = false) + List triggers = new ArrayList<>(); + + @PostConstruct + public void createSchedulers() { + + for (SchedulerConfig config : quartzMultiSchedulerConfig.getSchedulerConfigs()) { + Map quartzProps = config.getQuartz(); + Properties properties = new Properties(); + + for (Map.Entry entry : quartzProps.entrySet()) { + properties.put("org.quartz." + entry.getKey(), entry.getValue()); + logger.debug("org.quartz.{}={}", entry.getKey(), entry.getValue()); + } + SchedulerFactoryBean schedulerFactory = new SchedulerFactoryBean(); + applySchedulerFactoryProperties(schedulerFactory, properties); + + if (config.getName().equalsIgnoreCase(DEFAULT_SCHEDULER_NAME)) { + schedulerFactory.setTriggers(triggers.toArray(new Trigger[] {})); + } + + schedulerFactory.setBeanName(config.getName()); + + ConfigurableApplicationContext configurableApplicationContext = + (ConfigurableApplicationContext) applicationContext; + configurableApplicationContext + .getBeanFactory() + .initializeBean(schedulerFactory, config.getName()); + configurableApplicationContext.getBeanFactory().autowireBean(schedulerFactory); + configurableApplicationContext + .getBeanFactory() + .registerSingleton(config.getName(), schedulerFactory); + } + } + + public void applySchedulerFactoryProperties( + SchedulerFactoryBean schedulerFactory, Properties quartzProperties) { + schedulerFactory.setQuartzProperties(quartzProperties); + schedulerFactory.setJobFactory(createSpringBeanJobFactory()); + schedulerFactory.setOverwriteExistingJobs(true); + schedulerFactory.setAutoStartup(false); + + if (quartzMetricsReportingJobListener != null) { + schedulerFactory.setGlobalJobListeners(quartzMetricsReportingJobListener); + } + } + + public SpringBeanJobFactory createSpringBeanJobFactory() { + AutoWiringSpringBeanJobFactory jobFactory = new AutoWiringSpringBeanJobFactory(); + jobFactory.setApplicationContext(applicationContext); + return jobFactory; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerManager.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerManager.java new file mode 100644 index 0000000000..d216305560 --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/QuartzMultiSchedulerManager.java @@ -0,0 +1,63 @@ +package com.box.l10n.mojito.quartz.multi; + +import com.box.l10n.mojito.quartz.QuartzSchedulerManager; +import java.util.List; +import java.util.Map; +import javax.annotation.PostConstruct; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.DependsOn; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty(name = "l10n.org.multi-quartz.enabled", havingValue = "true") +@DependsOn("quartzMultiSchedulerFactory") +public class QuartzMultiSchedulerManager implements QuartzSchedulerManager { + + Logger logger = LoggerFactory.getLogger(QuartzMultiSchedulerManager.class); + + @Autowired List schedulers; + + Map schedulerMap = new java.util.HashMap<>(); + + @PostConstruct + public void init() throws SchedulerException { + for (Scheduler scheduler : schedulers) { + if (schedulerMap.containsKey(scheduler.getSchedulerName())) { + throw new QuartzMultiSchedulerException( + "Scheduler with name '" + + scheduler.getSchedulerName() + + "' already exists. Please configure a unique name for each scheduler"); + } + schedulerMap.put(scheduler.getSchedulerName(), scheduler); + } + + if (!schedulerMap.containsKey(DEFAULT_SCHEDULER_NAME)) { + throw new QuartzMultiSchedulerException( + "No default scheduler found. Please configure a scheduler with name '" + + DEFAULT_SCHEDULER_NAME + + "'"); + } + } + + @Override + public Scheduler getScheduler(String schedulerName) { + + if (!schedulerMap.containsKey(schedulerName)) { + logger.warn( + "Scheduler with name '{}' not found, scheduling job on default scheduler", schedulerName); + schedulerName = DEFAULT_SCHEDULER_NAME; + } + + return schedulerMap.get(schedulerName); + } + + @Override + public List getSchedulers() { + return schedulers; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/SchedulerConfig.java b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/SchedulerConfig.java new file mode 100644 index 0000000000..479a49658a --- /dev/null +++ b/webapp/src/main/java/com/box/l10n/mojito/quartz/multi/SchedulerConfig.java @@ -0,0 +1,35 @@ +package com.box.l10n.mojito.quartz.multi; + +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnProperty(name = "l10n.org.multi-quartz.enabled", havingValue = "true") +public class SchedulerConfig { + + Logger logger = LoggerFactory.getLogger(SchedulerConfig.class); + + private String name; + + private Map quartz = new HashMap<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map getQuartz() { + return quartz; + } + + public void setQuartz(Map quartz) { + this.quartz = quartz; + } +} diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/assetExtraction/AssetExtractionService.java b/webapp/src/main/java/com/box/l10n/mojito/service/assetExtraction/AssetExtractionService.java index d044fffe62..ff7470aa50 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/assetExtraction/AssetExtractionService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/assetExtraction/AssetExtractionService.java @@ -1,5 +1,6 @@ package com.box.l10n.mojito.service.assetExtraction; +import static com.box.l10n.mojito.quartz.QuartzSchedulerManager.DEFAULT_SCHEDULER_NAME; import static com.box.l10n.mojito.service.assetExtraction.LocalBranchToEntityBranchConverter.NULL_BRANCH_TEXT_PLACEHOLDER; import static java.util.function.Function.identity; @@ -83,6 +84,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.retry.RetryContext; import org.springframework.retry.annotation.Retryable; @@ -162,6 +164,9 @@ public class AssetExtractionService { @Autowired LocalBranchToEntityBranchConverter localBranchToEntityBranchConverter; + @Value("${l10n.assetExtraction.quartz.schedulerName:" + DEFAULT_SCHEDULER_NAME + "}") + String jobSchedulerName; + /** * If the asset type is supported, starts the text units extraction for the given asset. * @@ -1158,7 +1163,7 @@ public PollableFuture processAssetAsync( .withExpectedSubTaskNumber(5) .build(); - return quartzPollableTaskScheduler.scheduleJob(quartzJobInfo); + return quartzPollableTaskScheduler.scheduleJob(quartzJobInfo, jobSchedulerName); } /**