From af8a649a82b8b7542c6f4b770efb1b937c99925a Mon Sep 17 00:00:00 2001 From: Filipe Pina Date: Wed, 25 Jan 2023 01:59:43 +0000 Subject: [PATCH 1/7] add support for hash scheduling --- dkron/job.go | 40 ++++++++++++++++++++++++++++++++++++++-- dkron/scheduler.go | 2 +- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/dkron/job.go b/dkron/job.go index 8c498c6d3..d72806979 100644 --- a/dkron/job.go +++ b/dkron/job.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "regexp" + "strconv" + "strings" "time" "github.com/distribworks/dkron/v4/extcron" @@ -303,10 +305,44 @@ func (j *Job) GetTimeLocation() *time.Location { return loc } +// scheduleHash replaces H in the cron spec by a value derived from job Name +// such as "0 0 H * * *" +func (j *Job) scheduleHash() string { + spec := j.Schedule + if strings.Contains(spec, "H") && strings.Count(strings.TrimSpace(spec), " ") == 5 { + h := 0 + for _, c := range j.Name { + h += int(c) + } + parts := strings.Split(spec, " ") + for index, part := range parts { + if strings.Contains(part, "H") { + // mods taken in accordance with https://dkron.io/docs/usage/cron-spec/#cron-expression-format + ph := h + switch index { + case 2: + ph %= 24 + case 3: + ph = (ph % 31) + 1 + case 4: + ph = (ph % 12) + 1 + case 5: + ph %= 7 + default: + ph %= 60 + } + parts[index] = strings.ReplaceAll(part, "H", strconv.Itoa(ph)) + } + } + return strings.Join(parts, " ") + } + return spec +} + // GetNext returns the job's next schedule from now func (j *Job) GetNext() (time.Time, error) { if j.Schedule != "" { - s, err := extcron.Parse(j.Schedule) + s, err := extcron.Parse(j.scheduleHash()) if err != nil { return time.Time{}, err } @@ -367,7 +403,7 @@ func (j *Job) Validate() error { // Validate schedule, allow empty schedule if parent job set. if j.Schedule != "" || j.ParentJob == "" { - if _, err := extcron.Parse(j.Schedule); err != nil { + if _, err := extcron.Parse(j.scheduleHash()); err != nil { return fmt.Errorf("%s: %s", ErrScheduleParse.Error(), err) } } diff --git a/dkron/scheduler.go b/dkron/scheduler.go index ab62c380b..5a17879bd 100644 --- a/dkron/scheduler.go +++ b/dkron/scheduler.go @@ -159,7 +159,7 @@ func (s *Scheduler) AddJob(job *Job) error { // If Timezone is set on the job, and not explicitly in its schedule, // AND its not a descriptor (that don't support timezones), add the // timezone to the schedule so robfig/cron knows about it. - schedule := job.Schedule + schedule := job.scheduleHash() if job.Timezone != "" && !strings.HasPrefix(schedule, "@") && !strings.HasPrefix(schedule, "TZ=") && From f38f13515f6866b7863857702f4583b825ecea66 Mon Sep 17 00:00:00 2001 From: Filipe Pina Date: Sun, 11 Feb 2024 01:07:32 +0000 Subject: [PATCH 2/7] clearer variable names --- dkron/job.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dkron/job.go b/dkron/job.go index d72806979..e9aaf8dfc 100644 --- a/dkron/job.go +++ b/dkron/job.go @@ -310,28 +310,28 @@ func (j *Job) GetTimeLocation() *time.Location { func (j *Job) scheduleHash() string { spec := j.Schedule if strings.Contains(spec, "H") && strings.Count(strings.TrimSpace(spec), " ") == 5 { - h := 0 + hash := 0 for _, c := range j.Name { - h += int(c) + hash += int(c) } parts := strings.Split(spec, " ") for index, part := range parts { if strings.Contains(part, "H") { // mods taken in accordance with https://dkron.io/docs/usage/cron-spec/#cron-expression-format - ph := h + partHash := hash switch index { case 2: - ph %= 24 + partHash %= 24 case 3: - ph = (ph % 31) + 1 + partHash = (partHash % 31) + 1 case 4: - ph = (ph % 12) + 1 + partHash = (partHash % 12) + 1 case 5: - ph %= 7 + partHash %= 7 default: - ph %= 60 + partHash %= 60 } - parts[index] = strings.ReplaceAll(part, "H", strconv.Itoa(ph)) + parts[index] = strings.ReplaceAll(part, "H", strconv.Itoa(partHash)) } } return strings.Join(parts, " ") From 7552f19fcbf81ba9a2eab61fee0be5f4ad8b459a Mon Sep 17 00:00:00 2001 From: Filipe Pina Date: Sun, 11 Feb 2024 01:45:19 +0000 Subject: [PATCH 3/7] replace magic `H` with `~` --- dkron/job.go | 8 ++++---- dkron/job_test.go | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dkron/job.go b/dkron/job.go index e9aaf8dfc..2de8bf068 100644 --- a/dkron/job.go +++ b/dkron/job.go @@ -306,17 +306,17 @@ func (j *Job) GetTimeLocation() *time.Location { } // scheduleHash replaces H in the cron spec by a value derived from job Name -// such as "0 0 H * * *" +// such as "0 0 ~ * * *" func (j *Job) scheduleHash() string { spec := j.Schedule - if strings.Contains(spec, "H") && strings.Count(strings.TrimSpace(spec), " ") == 5 { + if strings.Contains(spec, "~") && strings.Count(strings.TrimSpace(spec), " ") == 5 { hash := 0 for _, c := range j.Name { hash += int(c) } parts := strings.Split(spec, " ") for index, part := range parts { - if strings.Contains(part, "H") { + if strings.Contains(part, "~") { // mods taken in accordance with https://dkron.io/docs/usage/cron-spec/#cron-expression-format partHash := hash switch index { @@ -331,7 +331,7 @@ func (j *Job) scheduleHash() string { default: partHash %= 60 } - parts[index] = strings.ReplaceAll(part, "H", strconv.Itoa(partHash)) + parts[index] = strings.ReplaceAll(part, "~", strconv.Itoa(partHash)) } } return strings.Join(parts, " ") diff --git a/dkron/job_test.go b/dkron/job_test.go index 22dc1746f..74179e8d1 100644 --- a/dkron/job_test.go +++ b/dkron/job_test.go @@ -192,6 +192,14 @@ func Test_isRunnable(t *testing.T) { } } +func Test_scheduleHash(t *testing.T) { + job := &Job{ + Name: "test_job", + Schedule: "0 0 ~ * * *", + } + assert.Equal(t, "0 0 18 * * *", job.scheduleHash()) +} + type gRPCClientMock struct { } From d3b24eaacada0d7c84b57033cbd340c952effad0 Mon Sep 17 00:00:00 2001 From: Filipe Pina Date: Sun, 11 Feb 2024 17:36:04 +0000 Subject: [PATCH 4/7] cleanup to handle `@`, and timezone variables --- dkron/job.go | 71 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/dkron/job.go b/dkron/job.go index 2de8bf068..20fd2428c 100644 --- a/dkron/job.go +++ b/dkron/job.go @@ -33,6 +33,9 @@ const ( ConcurrencyAllow = "allow" // ConcurrencyForbid forbids a job from executing concurrency. ConcurrencyForbid = "forbid" + + // HashSymbol is the "magic" character used in scheduled to be replaced with a value based on job name + HashSymbol = "~" ) var ( @@ -305,38 +308,58 @@ func (j *Job) GetTimeLocation() *time.Location { return loc } +// nameHash returns hash code of the job name +func (j *Job) nameHash() int { + hash := 0 + for _, c := range j.Name { + hash += int(c) + } + return hash +} + // scheduleHash replaces H in the cron spec by a value derived from job Name // such as "0 0 ~ * * *" func (j *Job) scheduleHash() string { spec := j.Schedule - if strings.Contains(spec, "~") && strings.Count(strings.TrimSpace(spec), " ") == 5 { - hash := 0 - for _, c := range j.Name { - hash += int(c) + + if !strings.Contains(spec, HashSymbol) { + return spec + } + + hash := j.nameHash() + parts := strings.Split(spec, " ") + partIndex := 0 + for index, part := range parts { + if strings.HasPrefix(part, "@") { + // this is a pre-defined scheduled, ignore everything + return spec } - parts := strings.Split(spec, " ") - for index, part := range parts { - if strings.Contains(part, "~") { - // mods taken in accordance with https://dkron.io/docs/usage/cron-spec/#cron-expression-format - partHash := hash - switch index { - case 2: - partHash %= 24 - case 3: - partHash = (partHash % 31) + 1 - case 4: - partHash = (partHash % 12) + 1 - case 5: - partHash %= 7 - default: - partHash %= 60 - } - parts[index] = strings.ReplaceAll(part, "~", strconv.Itoa(partHash)) + if strings.HasPrefix(part, "TZ=") || strings.HasPrefix(part, "CRON_TZ=") { + // do not increase partIndex + continue + } + + if strings.Contains(part, HashSymbol) { + // mods taken in accordance with https://dkron.io/docs/usage/cron-spec/#cron-expression-format + partHash := hash + switch partIndex { + case 2: + partHash %= 24 + case 3: + partHash = (partHash % 28) + 1 + case 4: + partHash = (partHash % 12) + 1 + case 5: + partHash %= 7 + default: + partHash %= 60 } + parts[index] = strings.ReplaceAll(part, HashSymbol, strconv.Itoa(partHash)) } - return strings.Join(parts, " ") + + partIndex++ } - return spec + return strings.Join(parts, " ") } // GetNext returns the job's next schedule from now From e28ea536241e04b3287c8b5dd919e33dcea024a7 Mon Sep 17 00:00:00 2001 From: Filipe Pina Date: Sun, 11 Feb 2024 17:52:29 +0000 Subject: [PATCH 5/7] test updated --- dkron/job_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dkron/job_test.go b/dkron/job_test.go index 74179e8d1..9dcf11114 100644 --- a/dkron/job_test.go +++ b/dkron/job_test.go @@ -194,10 +194,14 @@ func Test_isRunnable(t *testing.T) { func Test_scheduleHash(t *testing.T) { job := &Job{ - Name: "test_job", - Schedule: "0 0 ~ * * *", + Name: "test_job", } + job.Schedule = "0 0 ~ * * *" assert.Equal(t, "0 0 18 * * *", job.scheduleHash()) + job.Schedule = "TZ=Europe/Madrid 0 0 1 * ~ *" + assert.Equal(t, "TZ=Europe/Madrid 0 0 1 * 7 *", job.scheduleHash()) + job.Schedule = "TZ=Europe/Madrid @at something with ~" + assert.Equal(t, "TZ=Europe/Madrid @at something with ~", job.scheduleHash()) } type gRPCClientMock struct { From fe097d58b98689971d426c7136d79b78b8656024 Mon Sep 17 00:00:00 2001 From: Filipe Pina Date: Sun, 15 Sep 2024 11:16:27 +0100 Subject: [PATCH 6/7] docs updated --- website/docs/usage/cron-spec.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/website/docs/usage/cron-spec.md b/website/docs/usage/cron-spec.md index 77baf8025..7778c88ff 100644 --- a/website/docs/usage/cron-spec.md +++ b/website/docs/usage/cron-spec.md @@ -9,12 +9,12 @@ A cron expression represents a set of times, using 6 space-separated fields. Field name | Mandatory? | Allowed values | Allowed special characters ---------- | ---------- | -------------- | -------------------------- - Seconds | Yes | 0-59 | * / , - - Minutes | Yes | 0-59 | * / , - - Hours | Yes | 0-23 | * / , - - Day of month | Yes | 1-31 | * / , - ? - Month | Yes | 1-12 or JAN-DEC | * / , - - Day of week | Yes | 0-6 or SUN-SAT | * / , - ? + Seconds | Yes | 0-59 | * / , - ~ + Minutes | Yes | 0-59 | * / , - ~ + Hours | Yes | 0-23 | * / , - ~ + Day of month | Yes | 1-31 | * / , - ? ~ + Month | Yes | 1-12 or JAN-DEC | * / , - ~ + Day of week | Yes | 0-6 or SUN-SAT | * / , - ? ~ Note: Month and Day-of-week field values are case insensitive. "SUN", "Sun", and "sun" are equally accepted. @@ -51,6 +51,10 @@ Question mark ( ? ) Question mark may be used instead of '*' for leaving either day-of-month or day-of-week blank. +Tilde ( ~ ) + +Tilde will be replaced by a numeric value valid for the range where it is used. It allows periodically scheduled tasks to produce even load on the system. For example, scheduling multiple hourly jobs to "0 H * * * *" rather than "0 0 * * * *" will run the jobs at different minutes of every hour. It can be thought of as a random value over a range, but it actually is a hash of the job name, not a random function, so that the value remains stable for any given job. + ### Predefined schedules You may use one of several pre-defined schedules in place of a cron expression. From 927d9f8791f4f47bd4a6e59b655a2f696f5da70c Mon Sep 17 00:00:00 2001 From: Filipe Pina <636320+fopina@users.noreply.github.com> Date: Sun, 15 Sep 2024 12:37:04 +0100 Subject: [PATCH 7/7] typo --- website/docs/usage/cron-spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/usage/cron-spec.md b/website/docs/usage/cron-spec.md index 7778c88ff..13d3d2e23 100644 --- a/website/docs/usage/cron-spec.md +++ b/website/docs/usage/cron-spec.md @@ -53,7 +53,7 @@ day-of-week blank. Tilde ( ~ ) -Tilde will be replaced by a numeric value valid for the range where it is used. It allows periodically scheduled tasks to produce even load on the system. For example, scheduling multiple hourly jobs to "0 H * * * *" rather than "0 0 * * * *" will run the jobs at different minutes of every hour. It can be thought of as a random value over a range, but it actually is a hash of the job name, not a random function, so that the value remains stable for any given job. +Tilde will be replaced by a numeric value valid for the range where it is used. It allows periodically scheduled tasks to produce even load on the system. For example, scheduling multiple hourly jobs to "0 ~ * * * *" rather than "0 0 * * * *" will run the jobs at different minutes of every hour. It can be thought of as a random value over a range, but it actually is a hash of the job name, not a random function, so that the value remains stable for any given job. ### Predefined schedules