-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* Introduce async query scheduler Signed-off-by: Louis Chu <[email protected]> * Update IT and doc Signed-off-by: Louis Chu <[email protected]> * Resolve comments Signed-off-by: Louis Chu <[email protected]> * Bugfix Signed-off-by: Louis Chu <[email protected]> * Add more tests Signed-off-by: Louis Chu <[email protected]> --------- Signed-off-by: Louis Chu <[email protected]> (cherry picked from commit 92121a3)
- Loading branch information
Showing
38 changed files
with
1,721 additions
and
102 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
flint-commons/src/main/scala/org/opensearch/flint/common/scheduler/AsyncQueryScheduler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.flint.common.scheduler; | ||
|
|
||
|
|
||
| import org.opensearch.flint.common.scheduler.model.AsyncQuerySchedulerRequest; | ||
|
|
||
| /** Scheduler interface for scheduling asynchronous query jobs. Copied from https://github.com/opensearch-project/sql/blob/main/async-query-core/src/main/java/org/opensearch/sql/spark/scheduler/AsyncQueryScheduler.java */ | ||
| public interface AsyncQueryScheduler { | ||
|
|
||
| /** | ||
| * Schedules a new job in the system. This method creates a new job entry based on the provided | ||
| * request parameters. | ||
| * | ||
| * <p>Use cases: - Creating a new periodic query execution - Setting up a scheduled data refresh | ||
| * task | ||
| * | ||
| * @param asyncQuerySchedulerRequest The request containing job configuration details | ||
| * @throws IllegalArgumentException if a job with the same name already exists | ||
| * @throws RuntimeException if there's an error during job creation | ||
| */ | ||
| void scheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); | ||
|
|
||
| /** | ||
| * Updates an existing job with new parameters. This method modifies the configuration of an | ||
| * already scheduled job. | ||
| * | ||
| * <p>Use cases: - Changing the schedule of an existing job - Modifying query parameters of a | ||
| * scheduled job - Updating resource allocations for a job | ||
| * | ||
| * @param asyncQuerySchedulerRequest The request containing updated job configuration | ||
| * @throws IllegalArgumentException if the job to be updated doesn't exist | ||
| * @throws RuntimeException if there's an error during the update process | ||
| */ | ||
| void updateJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); | ||
|
|
||
| /** | ||
| * Unschedules a job by marking it as disabled and updating its last update time. This method is | ||
| * used when you want to temporarily stop a job from running but keep its configuration and | ||
| * history in the system. | ||
| * | ||
| * <p>Use cases: - Pausing a job that's causing issues without losing its configuration - | ||
| * Temporarily disabling a job during maintenance or high-load periods - Allowing for easy | ||
| * re-enabling of the job in the future | ||
| * | ||
| * @param asyncQuerySchedulerRequest The request containing the job configuration to unschedule. | ||
| * At minimum, it must include the jobId. | ||
| * @throws IllegalArgumentException if the job to be unscheduled doesn't exist | ||
| * @throws RuntimeException if there's an error during the unschedule process | ||
| */ | ||
| void unscheduleJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); | ||
|
|
||
| /** | ||
| * Removes a job completely from the scheduler. This method permanently deletes the job and all | ||
| * its associated data from the system. | ||
| * | ||
| * <p>Use cases: - Cleaning up jobs that are no longer needed - Removing obsolete or erroneously | ||
| * created jobs - Freeing up resources by deleting unused job configurations | ||
| * | ||
| * @param asyncQuerySchedulerRequest The request containing the job configuration to unschedule. | ||
| * At minimum, it must include the jobId. | ||
| * @throws IllegalArgumentException if the job to be removed doesn't exist | ||
| * @throws RuntimeException if there's an error during the remove process | ||
| */ | ||
| void removeJob(AsyncQuerySchedulerRequest asyncQuerySchedulerRequest); | ||
| } |
76 changes: 76 additions & 0 deletions
76
...rc/main/scala/org/opensearch/flint/common/scheduler/model/AsyncQuerySchedulerRequest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.flint.common.scheduler.model; | ||
|
|
||
| import lombok.Builder; | ||
| import lombok.Data; | ||
|
|
||
| import java.time.Instant; | ||
|
|
||
| /** | ||
| * Represents a job request for a scheduled task. | ||
| */ | ||
| @Builder | ||
| @Data | ||
| public class AsyncQuerySchedulerRequest { | ||
|
|
||
| /** | ||
| * The AWS accountId used to identify the resource. | ||
| */ | ||
| String accountId; | ||
|
|
||
| /** | ||
| * The unique identifier for the scheduler job. | ||
| */ | ||
| String jobId; | ||
|
|
||
| /** | ||
| * The name of the data source on which the scheduled query will be executed. | ||
| */ | ||
| String dataSource; | ||
|
|
||
| /** | ||
| * The scheduled query to be executed. | ||
| */ | ||
| String scheduledQuery; | ||
|
|
||
| /** | ||
| * The language in which the query is written, such as SQL, PPL (Piped Processing Language), etc. | ||
| */ | ||
| String queryLang; | ||
|
|
||
| /** | ||
| * The interval expression defining the frequency of the job execution. | ||
| * Typically expressed as a time-based pattern (e.g. 5 minutes). | ||
| */ | ||
| String interval; | ||
|
|
||
| /** | ||
| * Indicates whether the scheduled job is currently enabled or not. | ||
| */ | ||
| boolean enabled; | ||
|
|
||
| /** | ||
| * The timestamp of the last update made to this job. | ||
| */ | ||
| Instant lastUpdateTime; | ||
|
|
||
| /** | ||
| * The timestamp when this job was enabled. | ||
| */ | ||
| Instant enabledTime; | ||
|
|
||
| /** | ||
| * The duration, in seconds, for which the job remains locked. | ||
| * This lock is used to prevent concurrent executions of the same job, ensuring that only one instance of the job runs at a time. | ||
| */ | ||
| Long lockDurationSeconds; | ||
|
|
||
| /** | ||
| * The jitter value to add randomness to the execution schedule, helping to avoid the thundering herd problem. | ||
| */ | ||
| Double jitter; | ||
| } |
47 changes: 47 additions & 0 deletions
47
flint-core/src/main/resources/async-query-scheduler-index-mapping.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| --- | ||
| ## | ||
| # Copyright OpenSearch Contributors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| ## | ||
|
|
||
| # Schema file for the .async-query-scheduler index | ||
| # Also "dynamic" is set to "false" so that other fields cannot be added. | ||
| dynamic: false | ||
| properties: | ||
| accountId: | ||
| type: keyword | ||
| jobId: | ||
| type: keyword | ||
| dataSource: | ||
| type: keyword | ||
| scheduledQuery: | ||
| type: text | ||
| queryLang: | ||
| type: keyword | ||
| lastUpdateTime: | ||
| type: date | ||
| format: epoch_millis | ||
| enabledTime: | ||
| type: date | ||
| format: epoch_millis | ||
| schedule: | ||
| properties: | ||
| initialDelay: | ||
| type: long | ||
| interval: | ||
| properties: | ||
| start_time: | ||
| type: date | ||
| format: "strict_date_time||epoch_millis" | ||
| period: | ||
| type: integer | ||
| unit: | ||
| type: keyword | ||
| enabled: | ||
| type: boolean | ||
| lockDurationSeconds: | ||
| type: long | ||
| null_value: -1 | ||
| jitter: | ||
| type: double | ||
| null_value: 0.0 |
11 changes: 11 additions & 0 deletions
11
flint-core/src/main/resources/async-query-scheduler-index-settings.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| --- | ||
| ## | ||
| # Copyright OpenSearch Contributors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| ## | ||
|
|
||
| # Settings file for the .async-query-scheduler index | ||
| index: | ||
| number_of_shards: "1" | ||
| auto_expand_replicas: "0-2" | ||
| number_of_replicas: "0" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.