From c984d54ca09c9ca3415258a398398388dd22ef63 Mon Sep 17 00:00:00 2001 From: Kamil Samigullin Date: Sun, 19 Jul 2020 16:41:55 +0300 Subject: [PATCH] complete specification of app.toml --- internal/model/merge.go | 94 ++++++----- internal/model/merge_test.go | 13 +- internal/model/specification.go | 44 +++-- internal/model/specification_test.go | 38 +++++ internal/model/testdata/app.toml | 27 ++- .../model/testdata/components/envs/base.toml | 156 ++++++++++-------- .../model/testdata/components/envs/perf.toml | 21 ++- .../model/testdata/components/envs/prod.toml | 36 ++-- internal/model/testdata/dependencies.toml | 16 +- 9 files changed, 290 insertions(+), 155 deletions(-) diff --git a/internal/model/merge.go b/internal/model/merge.go index 703aa51..462fdea 100644 --- a/internal/model/merge.go +++ b/internal/model/merge.go @@ -39,15 +39,15 @@ func (crons *Crons) Merge(src Crons) { copied := *crons copied = append(copied, src...) sort.Sort(copied) - j := 0 + shift := 0 for i := 1; i < len(copied); i++ { - if copied[j].Name == copied[i].Name { + if copied[shift].Name == copied[i].Name { continue } - j++ - copied[j] = copied[i] + shift++ + copied[shift] = copied[i] } - *crons = copied[:j+1] + *crons = copied[:shift+1] } func (deps *Dependencies) Merge(src Dependencies) { @@ -58,15 +58,15 @@ func (deps *Dependencies) Merge(src Dependencies) { copied := *deps copied = append(copied, src...) sort.Sort(copied) - j := 0 + shift := 0 for i := 1; i < len(copied); i++ { - if copied[j].Name == copied[i].Name { + if copied[shift].Name == copied[i].Name { continue } - j++ - copied[j] = copied[i] + shift++ + copied[shift] = copied[i] } - *deps = copied[:j+1] + *deps = copied[:shift+1] } func (engine *Engine) Merge(src *Engine) { @@ -111,15 +111,15 @@ func (exec *Executable) Merge(src Executable) { copied := *exec copied = append(copied, src...) sort.Sort(copied) - j := 0 + shift := 0 for i := 1; i < len(copied); i++ { - if copied[j].Name == copied[i].Name { + if copied[shift].Name == copied[i].Name { continue } - j++ - copied[j] = copied[i] + shift++ + copied[shift] = copied[i] } - *exec = copied[:j+1] + *exec = copied[:shift+1] } func (logger *Logger) Merge(src *Logger) { @@ -140,15 +140,15 @@ func (proxies *Proxies) Merge(src Proxies) { copied := *proxies copied = append(copied, src...) sort.Sort(copied) - j := 0 + shift := 0 for i := 1; i < len(copied); i++ { - if copied[j].Name == copied[i].Name { + if copied[shift].Name == copied[i].Name { continue } - j++ - copied[j] = copied[i] + shift++ + copied[shift] = copied[i] } - *proxies = copied[:j+1] + *proxies = copied[:shift+1] } func (queues *Queues) Merge(src Queues) { @@ -159,15 +159,15 @@ func (queues *Queues) Merge(src Queues) { copied := *queues copied = append(copied, src...) sort.Sort(copied) - j := 0 + shift := 0 for i := 1; i < len(copied); i++ { - if copied[j].Name == copied[i].Name { + if copied[shift].Name == copied[i].Name { continue } - j++ - copied[j] = copied[i] + shift++ + copied[shift] = copied[i] } - *queues = copied[:j+1] + *queues = copied[:shift+1] } func (resource *Resource) Merge(src *Resource) { @@ -199,6 +199,19 @@ func (resources *Resources) Merge(src *Resources) { resources.Limits.Merge(src.Limits) } +func (sftp *SFTP) Merge(src *SFTP) { + if sftp == nil || src == nil { + return + } + + if src.Size != "" { + sftp.Size = src.Size + } + if src.Enabled != nil { + sftp.Enabled = src.Enabled + } +} + func (spec *Specification) Merge(src *Specification) { if spec == nil || src == nil { return @@ -245,21 +258,28 @@ func (spec *Specification) Merge(src *Specification) { spec.Executable.Merge(src.Executable) spec.Proxies.Merge(src.Proxies) spec.Queues.Merge(src.Queues) + spec.Sphinxes.Merge(src.Sphinxes) spec.Workers.Merge(src.Workers) spec.EnvVars.Merge(src.EnvVars) } -func (sftp *SFTP) Merge(src *SFTP) { - if sftp == nil || src == nil { +func (sphinxes *Sphinxes) Merge(src Sphinxes) { + if sphinxes == nil || len(src) == 0 { return } - if src.Size != "" { - sftp.Size = src.Size - } - if src.Enabled != nil { - sftp.Enabled = src.Enabled + copied := *sphinxes + copied = append(copied, src...) + sort.Sort(copied) + shift := 0 + for i := 1; i < len(copied); i++ { + if copied[shift].Name == copied[i].Name { + continue + } + shift++ + copied[shift] = copied[i] } + *sphinxes = copied[:shift+1] } func (workers *Workers) Merge(src Workers) { @@ -270,13 +290,13 @@ func (workers *Workers) Merge(src Workers) { copied := *workers copied = append(copied, src...) sort.Sort(copied) - j := 0 + shift := 0 for i := 1; i < len(copied); i++ { - if copied[j].Name == copied[i].Name { + if copied[shift].Name == copied[i].Name { continue } - j++ - copied[j] = copied[i] + shift++ + copied[shift] = copied[i] } - *workers = copied[:j+1] + *workers = copied[:shift+1] } diff --git a/internal/model/merge_test.go b/internal/model/merge_test.go index 04e2313..c802fa7 100644 --- a/internal/model/merge_test.go +++ b/internal/model/merge_test.go @@ -117,15 +117,20 @@ func TestMerge(t *testing.T) { assert.NotPanics(t, func() { resources.Merge(&Resources{Requests: &Resource{CPU: 1}}) }) assert.Nil(t, resources) }) + t.Run("nil sftp", func(t *testing.T) { + var sftp *SFTP + assert.NotPanics(t, func() { sftp.Merge(&SFTP{Size: "small"}) }) + assert.Nil(t, sftp) + }) t.Run("nil specification", func(t *testing.T) { var spec *Specification assert.NotPanics(t, func() { spec.Merge(&Specification{Name: "test"}) }) assert.Nil(t, spec) }) - t.Run("nil sftp", func(t *testing.T) { - var sftp *SFTP - assert.NotPanics(t, func() { sftp.Merge(&SFTP{Size: "small"}) }) - assert.Nil(t, sftp) + t.Run("nil sphinxes", func(t *testing.T) { + var sphinxes *Sphinxes + assert.NotPanics(t, func() { sphinxes.Merge(Sphinxes{{Name: "test"}}) }) + assert.Nil(t, sphinxes) }) t.Run("nil workers", func(t *testing.T) { var workers *Workers diff --git a/internal/model/specification.go b/internal/model/specification.go index 891bba7..b04dd1c 100644 --- a/internal/model/specification.go +++ b/internal/model/specification.go @@ -47,7 +47,7 @@ type Exec struct { Name string `toml:"name,omitempty"` Replicas uint `toml:"replicas,omitempty"` Command string `toml:"command,omitempty"` - Port int `toml:"service-port,omitempty"` + Port uint `toml:"service-port,omitempty"` Size string `toml:"size,omitempty"` RedinessProbe string `toml:"readiness-probe-command,omitempty"` LivenessProbe string `toml:"liveness-probe-command,omitempty"` @@ -61,7 +61,9 @@ func (exec Executable) Swap(i, j int) { exec[i], exec[j] = exec[j], exec[i] type Host struct { Name string `toml:"host,omitempty"` + AgentPort uint `toml:"agent_port,omitempty"` Connections uint `toml:"connections,omitempty"` + MaxConns uint `toml:"maxconn,omitempty"` Weight uint `toml:"weight,omitempty"` Backup bool `toml:"backup,omitempty"` } @@ -101,8 +103,8 @@ func (queues Queues) Less(i, j int) bool { return queues[i].Name < queues[j].Nam func (queues Queues) Swap(i, j int) { queues[i], queues[j] = queues[j], queues[i] } type Resource struct { - CPU int `toml:"cpu,omitempty"` - Memory int `toml:"memory,omitempty"` + CPU uint `toml:"cpu,omitempty"` + Memory uint `toml:"memory,omitempty"` } type Resources struct { @@ -110,6 +112,11 @@ type Resources struct { Limits *Resource `toml:"limits,omitempty"` } +type SFTP struct { + Size string `toml:"size,omitempty"` + Enabled *bool `toml:"enabled,omitempty"` +} + type Specification struct { Name string `toml:"name,omitempty"` Description string `toml:"kind,omitempty"` @@ -125,24 +132,33 @@ type Specification struct { Executable Executable `toml:"executable,omitempty"` Proxies Proxies `toml:"proxy,omitempty"` Queues Queues `toml:"queues,omitempty"` + Sphinxes Sphinxes `toml:"sphinx,omitempty"` Workers Workers `toml:"workers,omitempty"` EnvVars EnvironmentVariables `toml:"env_vars,omitempty"` } -type SFTP struct { - Size string `toml:"size,omitempty"` - Enabled *bool `toml:"enabled,omitempty"` +type Sphinx struct { + Name string `toml:"name,omitepmty"` + Enabled *bool `toml:"enabled,omitepmty"` + Haproxy string `toml:"haproxy_tag,omitempty"` + Hosts Hosts `toml:"hosts,omitempty"` } +type Sphinxes []Sphinx + +func (sphinxes Sphinxes) Len() int { return len(sphinxes) } +func (sphinxes Sphinxes) Less(i, j int) bool { return sphinxes[i].Name < sphinxes[j].Name } +func (sphinxes Sphinxes) Swap(i, j int) { sphinxes[i], sphinxes[j] = sphinxes[j], sphinxes[i] } + type Worker struct { - Name string `toml:"name,omitempty"` - Enabled *bool `toml:"enabled,omitempty"` - Replicas uint `toml:"replicas,omitempty"` - Command string `toml:"command,omitempty"` - Commands []string `toml:"commands,omitempty"` - Size string `toml:"size,omitempty"` - LivenessProbe string `toml:"liveness-probe-command,omitempty"` - Resources Resources `toml:"resources,omitempty"` + Name string `toml:"name,omitempty"` + Enabled *bool `toml:"enabled,omitempty"` + Replicas uint `toml:"replicas,omitempty"` + Command string `toml:"command,omitempty"` + Commands []string `toml:"commands,omitempty"` + Size string `toml:"size,omitempty"` + LivenessProbe string `toml:"liveness-probe-command,omitempty"` + Resources *Resources `toml:"resources,omitempty"` } type Workers []Worker diff --git a/internal/model/specification_test.go b/internal/model/specification_test.go index d2e6f6f..181fe56 100644 --- a/internal/model/specification_test.go +++ b/internal/model/specification_test.go @@ -278,6 +278,44 @@ func TestSorting(t *testing.T) { }) } }) + t.Run("Sphinxes", func(t *testing.T) { + tests := map[string]struct { + input Sphinxes + expected Sphinxes + }{ + "sorted": { + input: Sphinxes{ + {Name: "a"}, + {Name: "b"}, + {Name: "c"}, + }, + expected: Sphinxes{ + {Name: "a"}, + {Name: "b"}, + {Name: "c"}, + }, + }, + "unsorted": { + input: Sphinxes{ + {Name: "b"}, + {Name: "c"}, + {Name: "a"}, + }, + expected: Sphinxes{ + {Name: "a"}, + {Name: "b"}, + {Name: "c"}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + sort.Sort(test.input) + assert.Equal(t, test.expected, test.input) + }) + } + }) t.Run("Workers", func(t *testing.T) { tests := map[string]struct { input Workers diff --git a/internal/model/testdata/app.toml b/internal/model/testdata/app.toml index 79bfbd1..4dea494 100644 --- a/internal/model/testdata/app.toml +++ b/internal/model/testdata/app.toml @@ -1,5 +1,4 @@ description = "Some awesome service" -host = "http://www.example.com" kind = "business" name = "service" replicas = 1 @@ -41,6 +40,7 @@ replicas = 1 [envs] [envs.perf] + host = "http://www.example.com" [[envs.perf.crons]] enabled = false @@ -75,6 +75,10 @@ replicas = 1 [envs.perf.sftp] enabled = false + [[envs.perf.sphinx]] + enabled = false + name = "search" + [envs.prod] [envs.prod.engine] @@ -169,9 +173,30 @@ replicas = 1 schema = "item.landed.on.mars" [[queues]] + aliases = ["item.landed.report.1", "item.landed.report.2", "item.landed.report.3"] dlq = ["5m", "10m", "30m"] schema = "item.landed.on.moon" [sftp] enabled = true size = "small" + +[[sphinx]] + enabled = true + name = "search" + + [[sphinx.hosts]] + host = "search01:9306" + + [[sphinx.hosts]] + host = "search02:9306" + +[[sphinx]] + enabled = true + name = "standby" + + [[sphinx.hosts]] + host = "search03:9306" + + [[sphinx.hosts]] + host = "search04:9306" diff --git a/internal/model/testdata/components/envs/base.toml b/internal/model/testdata/components/envs/base.toml index dd22d40..e530e36 100644 --- a/internal/model/testdata/components/envs/base.toml +++ b/internal/model/testdata/components/envs/base.toml @@ -1,104 +1,128 @@ name = "service" kind = "business" description = "Some awesome service" -host = "http://www.example.com" replicas = 1 [engine] -name = "golang" -version = "1.14" -size = "medium" + name = "golang" + version = "1.14" + size = "medium" [balancing] -cookie_affinity = "u" + cookie_affinity = "u" [[crons]] -name = "exporter" -enabled = true -schedule = "11 17 * * *" -command = "/bin/exporter --count 20" + name = "exporter" + enabled = true + schedule = "11 17 * * *" + command = "/bin/exporter --count 20" [[crons]] -name = "exporter" -enabled = true -schedule = "11 17 * * *" -command = "/bin/exporter --count 20" + name = "exporter" + enabled = true + schedule = "11 17 * * *" + command = "/bin/exporter --count 20" [[crons]] -name = "sampler" -enabled = true -schedule = "11 17 * * *" -command = "/bin/sampler --count 20" + name = "sampler" + enabled = true + schedule = "11 17 * * *" + command = "/bin/sampler --count 20" [[dependencies]] -name = "service-x" + name = "service-x" [[executable]] -name = "daemon" -replicas = 1 -readiness-probe-command = "curl --fail http://127.0.0.1:4321" -liveness-probe-command = "curl --fail http://127.0.0.1:4321" -command = "/usr/bin/daemon listen -at 4321" -service-port = 4321 -size = "small" + name = "daemon" + replicas = 1 + readiness-probe-command = "curl --fail http://127.0.0.1:4321" + liveness-probe-command = "curl --fail http://127.0.0.1:4321" + command = "/usr/bin/daemon listen -at 4321" + service-port = 4321 + size = "small" [[executable]] -name = "watcher" -replicas = 1 -readiness-probe-command = "curl --fail http://127.0.0.1:1234" -liveness-probe-command = "curl --fail http://127.0.0.1:1234" -command = "/usr/bin/watcher watch -at 1234" -service-port = 1234 -size = "small" + name = "watcher" + replicas = 1 + readiness-probe-command = "curl --fail http://127.0.0.1:1234" + liveness-probe-command = "curl --fail http://127.0.0.1:1234" + command = "/usr/bin/watcher watch -at 1234" + service-port = 1234 + size = "small" [[executable]] -name = "watcher" -replicas = 1 -readiness-probe-command = "curl --fail http://127.0.0.1:1234" -liveness-probe-command = "curl --fail http://127.0.0.1:1234" -command = "/usr/bin/watcher watch -at 1234" -service-port = 1234 -size = "small" + name = "watcher" + replicas = 1 + readiness-probe-command = "curl --fail http://127.0.0.1:1234" + liveness-probe-command = "curl --fail http://127.0.0.1:1234" + command = "/usr/bin/watcher watch -at 1234" + service-port = 1234 + size = "small" [logger] -level = "info" + level = "info" [[proxy]] -name = "s3" -enabled = true -hosts = [ - {host = "gateway01:80", connections = 50, weight = 100}, - {host = "gateway02:80", connections = 50, weight = 100, backup = true}, -] + name = "s3" + enabled = true + hosts = [ + {host = "gateway01:80", connections = 50, weight = 100}, + {host = "gateway02:80", connections = 50, weight = 100, backup = true}, + ] [[proxy]] -name = "s3" -enabled = true -hosts = [ - {host = "gateway01:80", connections = 50, weight = 100}, - {host = "gateway02:80", connections = 50, weight = 100, backup = true}, -] + name = "s3" + enabled = true + hosts = [ + {host = "gateway01:80", connections = 50, weight = 100}, + {host = "gateway02:80", connections = 50, weight = 100, backup = true}, + ] [[proxy]] -name = "s4" -enabled = true -hosts = [ - {host = "gateway01:80", connections = 50, weight = 100}, - {host = "gateway02:80", connections = 50, weight = 100, backup = true}, -] + name = "s4" + enabled = true + hosts = [ + {host = "gateway01:80", connections = 50, weight = 100}, + {host = "gateway02:80", connections = 50, weight = 100, backup = true}, + ] [[queues]] -schema = "item.landed.on.mars" -dlq=["5m", "10m", "30m"] + schema = "item.landed.on.mars" + dlq = ["5m", "10m", "30m"] [[queues]] -schema = "item.landed.on.mars" -dlq=["5m", "10m", "30m"] + schema = "item.landed.on.mars" + dlq = ["5m", "10m", "30m"] [[queues]] -schema = "item.landed.on.moon" -dlq=["5m", "10m", "30m"] + schema = "item.landed.on.moon" + dlq = ["5m", "10m", "30m"] + aliases = ["item.landed.report.1", "item.landed.report.2", "item.landed.report.3"] [sftp] -enabled = true -size = "small" + enabled = true + size = "small" + +[[sphinx]] + name = "search" + enabled = true + hosts = [ + {host = "search01:9306"}, + {host = "search02:9306"}, + ] + +[[sphinx]] + name = "search" + enabled = true + hosts = [ + {host = "search01:9306"}, + {host = "search02:9306"}, + ] + +[[sphinx]] + name = "standby" + enabled = true + hosts = [ + {host = "search03:9306"}, + {host = "search04:9306"}, + ] diff --git a/internal/model/testdata/components/envs/perf.toml b/internal/model/testdata/components/envs/perf.toml index f5090e5..e020371 100644 --- a/internal/model/testdata/components/envs/perf.toml +++ b/internal/model/testdata/components/envs/perf.toml @@ -1,14 +1,21 @@ +[envs.perf] + host = "http://www.example.com" + [[envs.perf.crons]] -name = "exporter" -enabled = false + name = "exporter" + enabled = false [[envs.perf.crons]] -name = "sampler" -enabled = false + name = "sampler" + enabled = false [[envs.perf.proxy]] -name = "s3" -enabled = false + name = "s3" + enabled = false [envs.perf.sftp] -enabled = false + enabled = false + +[[envs.perf.sphinx]] + name = "search" + enabled = false diff --git a/internal/model/testdata/components/envs/prod.toml b/internal/model/testdata/components/envs/prod.toml index 2c5f8c4..d4f50a9 100644 --- a/internal/model/testdata/components/envs/prod.toml +++ b/internal/model/testdata/components/envs/prod.toml @@ -1,27 +1,27 @@ [envs.prod.engine] -size = "large" -resources.requests.cpu = 1000 -resources.requests.memory = 9 -resources.limits.cpu = 10000 -resources.limits.memory = 99 + size = "large" + resources.requests.cpu = 1000 + resources.requests.memory = 9 + resources.limits.cpu = 10000 + resources.limits.memory = 99 [envs.prod.logger] -level = "warning" + level = "warning" [[envs.prod.workers]] -name = "memes-downloader" -replicas = 1 -command = "php /app/workers/get-memes.php" -size = "small" + name = "memes-downloader" + replicas = 1 + command = "php /app/workers/get-memes.php" + size = "small" [[envs.prod.workers]] -name = "memes-downloader" -replicas = 1 -command = "php /app/workers/get-memes.php" -size = "small" + name = "memes-downloader" + replicas = 1 + command = "php /app/workers/get-memes.php" + size = "small" [[envs.prod.workers]] -name = "resizer" -replicas = 1 -command = "/usr/bin/resizer start" -size = "small" + name = "resizer" + replicas = 1 + command = "/usr/bin/resizer start" + size = "small" diff --git a/internal/model/testdata/dependencies.toml b/internal/model/testdata/dependencies.toml index 4fd08d5..01fa69c 100644 --- a/internal/model/testdata/dependencies.toml +++ b/internal/model/testdata/dependencies.toml @@ -1,15 +1,15 @@ [[dependencies]] -name = "service-x" + name = "service-x" [[envs.perf.dependencies]] -name = "service-x" -mock = true -mock-replicas = 1 + name = "service-x" + mock = true + mock-replicas = 1 [[dependencies]] -name = "service-y" + name = "service-y" [[envs.perf.dependencies]] -name = "service-y" -mock = true -mock-replicas = 1 + name = "service-y" + mock = true + mock-replicas = 1