From a6308edd52f3d6b0699fabb6c40a397bc42c76a1 Mon Sep 17 00:00:00 2001 From: Hamish Moffatt <7577172+hmoffatt@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:28:56 +0000 Subject: [PATCH 1/3] DOCS: Add instructions for increasing the timeout on integration tests (#3233) Co-authored-by: Tom Limoncelli --- documentation/integration-tests.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/documentation/integration-tests.md b/documentation/integration-tests.md index a0c7baee98..9de0291345 100644 --- a/documentation/integration-tests.md +++ b/documentation/integration-tests.md @@ -47,6 +47,14 @@ go test -v -verbose -provider ROUTE53 -end 5 go test -v -verbose -provider ROUTE53 -start 16 -end 20 ``` +For some providers it may be necessary to increase the test timeout using `-test`. The default is 10 minutes. `0` is "no limit". Typical Go durations work too (`1h` for 1 hour, etc). + +```shell +go test -timeout 0 -v -verbose -provider CLOUDNS +``` + +FYI: The order of the flags matters. Flags native to the Go testing suite (`-timeout` and `-v`) must come before flags that are part of the DNSControl integration tests (`-verbose`, `-provider`). Yeah, that sucks and is confusing. + The actual tests are in the file `integrationTest/integration_test.go`. The tests are in a little language which can be used to describe just about any interaction with the API. Look for the comment `START HERE` or the line From 5d625e81ea097c5fc15ac244514127e83da2aa28 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 12 Dec 2024 18:07:49 +0100 Subject: [PATCH 2/3] HETZNER: enable concurrency capability (#3234) Signed-off-by: Jakob Ackermann Co-authored-by: Tom Limoncelli --- documentation/providers.md | 2 +- providers/hetzner/api.go | 46 ++++++++++++++++++++-------- providers/hetzner/hetznerProvider.go | 21 +++++++------ 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/documentation/providers.md b/documentation/providers.md index 111d55f0c7..378d3c7790 100644 --- a/documentation/providers.md +++ b/documentation/providers.md @@ -38,7 +38,7 @@ If a feature is definitively not supported for whatever reason, we would also li | [`GCLOUD`](provider/gcloud.md) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | ✅ | ❌ | ❔ | ✅ | ❔ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | | [`GCORE`](provider/gcore.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❔ | ✅ | ❌ | ✅ | ❌ | ❌ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | | [`HEDNS`](provider/hedns.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | -| [`HETZNER`](provider/hetzner.md) | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❔ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❔ | ✅ | ✅ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | +| [`HETZNER`](provider/hetzner.md) | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❔ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❔ | ✅ | ✅ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | | [`HEXONET`](provider/hexonet.md) | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ✅ | ❔ | | [`HOSTINGDE`](provider/hostingde.md) | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ | ✅ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | | [`HUAWEICLOUD`](provider/huaweicloud.md) | ❌ | ✅ | ❌ | ❔ | ❌ | ✅ | ❔ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | diff --git a/providers/hetzner/api.go b/providers/hetzner/api.go index 52e3dc7f4a..8efcd449e2 100644 --- a/providers/hetzner/api.go +++ b/providers/hetzner/api.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "strconv" + "sync" "time" "github.com/StackExchange/dnscontrol/v4/pkg/printer" @@ -18,7 +19,8 @@ const ( type hetznerProvider struct { apiKey string - zones map[string]zone + mu sync.Mutex + cachedZones map[string]zone requestRateLimiter requestRateLimiter } @@ -103,9 +105,17 @@ func (api *hetznerProvider) getAllRecords(domain string) ([]record, error) { return records, nil } -func (api *hetznerProvider) getAllZones() error { - if api.zones != nil { - return nil +func (api *hetznerProvider) resetZoneCache() { + api.mu.Lock() + defer api.mu.Unlock() + api.cachedZones = nil +} + +func (api *hetznerProvider) getAllZones() (map[string]zone, error) { + api.mu.Lock() + defer api.mu.Unlock() + if api.cachedZones != nil { + return api.cachedZones, nil } var zones map[string]zone page := 1 @@ -124,7 +134,7 @@ func (api *hetznerProvider) getAllZones() error { response := getAllZonesResponse{} url := fmt.Sprintf("/zones?per_page=100&page=%d", page) if err := api.request(url, "GET", nil, &response, statusOK); err != nil { - return fmt.Errorf("failed fetching zones: %w", err) + return nil, fmt.Errorf("failed fetching zones: %w", err) } if zones == nil { zones = make(map[string]zone, response.Meta.Pagination.TotalEntries) @@ -138,15 +148,16 @@ func (api *hetznerProvider) getAllZones() error { } page++ } - api.zones = zones - return nil + api.cachedZones = zones + return zones, nil } func (api *hetznerProvider) getZone(name string) (*zone, error) { - if err := api.getAllZones(); err != nil { + zones, err := api.getAllZones() + if err != nil { return nil, err } - z, ok := api.zones[name] + z, ok := zones[name] if !ok { return nil, fmt.Errorf("%q is not a zone in this HETZNER account", name) } @@ -213,18 +224,28 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte } type requestRateLimiter struct { + mu sync.Mutex delay time.Duration lastRequest time.Time + resetAt time.Time } func (rrl *requestRateLimiter) delayRequest() { - time.Sleep(time.Until(rrl.lastRequest.Add(rrl.delay))) - + rrl.mu.Lock() // When not rate-limited, include network/server latency in delay. - rrl.lastRequest = time.Now() + next := rrl.lastRequest.Add(rrl.delay) + if next.After(rrl.resetAt) { + // Do not stack delays past the reset point. + next = rrl.resetAt + } + rrl.lastRequest = next + rrl.mu.Unlock() + time.Sleep(time.Until(next)) } func (rrl *requestRateLimiter) handleResponse(resp *http.Response) (bool, error) { + rrl.mu.Lock() + defer rrl.mu.Unlock() if resp.StatusCode == http.StatusTooManyRequests { printer.Printf("Rate-Limited. Consider contacting the Hetzner Support for raising your quota. URL: %q, Headers: %q\n", resp.Request.URL, resp.Header) @@ -264,5 +285,6 @@ func (rrl *requestRateLimiter) handleResponse(resp *http.Response) (bool, error) // ... then spread requests evenly throughout the window. rrl.delay = reset / time.Duration(remaining+1) } + rrl.resetAt = time.Now().Add(reset) return false, nil } diff --git a/providers/hetzner/hetznerProvider.go b/providers/hetzner/hetznerProvider.go index 8f911919d8..414b2ecfc6 100644 --- a/providers/hetzner/hetznerProvider.go +++ b/providers/hetzner/hetznerProvider.go @@ -15,7 +15,7 @@ var features = providers.DocumentationNotes{ // See providers/capabilities.go for the entire list of capabilities. providers.CanAutoDNSSEC: providers.Cannot(), providers.CanGetZones: providers.Can(), - providers.CanConcur: providers.Cannot(), + providers.CanConcur: providers.Can(), providers.CanUseAlias: providers.Cannot(), providers.CanUseCAA: providers.Can(), providers.CanUseDS: providers.Can(), @@ -68,9 +68,11 @@ func (api *hetznerProvider) EnsureZoneExists(domain string) error { } } - // reset zone cache - api.zones = nil - return api.createZone(domain) + if err = api.createZone(domain); err != nil { + return err + } + api.resetZoneCache() + return nil } // GetZoneRecordsCorrections returns a list of corrections that will turn existing records into dc.Records. @@ -167,12 +169,13 @@ func (api *hetznerProvider) GetZoneRecords(domain string, meta map[string]string // ListZones lists the zones on this account. func (api *hetznerProvider) ListZones() ([]string, error) { - if err := api.getAllZones(); err != nil { + zones, err := api.getAllZones() + if err != nil { return nil, err } - zones := make([]string, 0, len(api.zones)) - for domain := range api.zones { - zones = append(zones, domain) + domains := make([]string, 0, len(zones)) + for domain := range zones { + domains = append(domains, domain) } - return zones, nil + return domains, nil } From 9d42930120aad369a0559e32eeb8bba4035a3815 Mon Sep 17 00:00:00 2001 From: Hamish Moffatt <7577172+hmoffatt@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:10:38 +0000 Subject: [PATCH 3/3] CLOUDNS: enable concurrent access (#3232) Co-authored-by: Tom Limoncelli --- documentation/providers.md | 2 +- go.mod | 4 +- providers/cloudns/api.go | 120 +++++++++++++++------------ providers/cloudns/cloudnsProvider.go | 38 +++++---- 4 files changed, 89 insertions(+), 75 deletions(-) diff --git a/documentation/providers.md b/documentation/providers.md index 378d3c7790..5d386c8b31 100644 --- a/documentation/providers.md +++ b/documentation/providers.md @@ -22,7 +22,7 @@ If a feature is definitively not supported for whatever reason, we would also li | [`BIND`](provider/bind.md) | ✅ | ✅ | ❌ | ❌ | ❔ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | [`BUNNY_DNS`](provider/bunny_dns.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❔ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❔ | ❌ | ❌ | ❌ | ❔ | ❔ | ❌ | ✅ | ✅ | | [`CLOUDFLAREAPI`](provider/cloudflareapi.md) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | ✅ | ❌ | ✅ | ✅ | ❔ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | ❔ | ❌ | ❌ | ✅ | ✅ | -| [`CLOUDNS`](provider/cloudns.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❔ | ✅ | ❔ | ✅ | ❔ | ✅ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ✅ | ✅ | +| [`CLOUDNS`](provider/cloudns.md) | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ | ❔ | ✅ | ❔ | ✅ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ✅ | ✅ | | [`CNR`](provider/cnr.md) | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ | | [`CSCGLOBAL`](provider/cscglobal.md) | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❌ | ✅ | | [`DESEC`](provider/desec.md) | ❌ | ✅ | ❌ | ✅ | ❔ | ✅ | ✅ | ✅ | ❔ | ✅ | ✅ | ❔ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | ✅ | ❔ | ✅ | ✅ | diff --git a/go.mod b/go.mod index 7639260ce6..caa03c146c 100644 --- a/go.mod +++ b/go.mod @@ -59,8 +59,8 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 github.com/G-Core/gcore-dns-sdk-go v0.2.9 - github.com/containrrr/shoutrrr v0.8.0 github.com/centralnicgroup-opensource/rtldev-middleware-go-sdk/v5 v5.0.3 + github.com/containrrr/shoutrrr v0.8.0 github.com/fatih/color v1.18.0 github.com/fbiville/markdown-table-formatter v0.3.0 github.com/go-acme/lego/v4 v4.20.4 @@ -74,6 +74,7 @@ require ( github.com/vultr/govultr/v2 v2.17.2 golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d golang.org/x/text v0.21.0 + golang.org/x/time v0.8.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -158,7 +159,6 @@ require ( golang.org/x/mod v0.22.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect - golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.28.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241118233622-e639e219e697 // indirect diff --git a/providers/cloudns/api.go b/providers/cloudns/api.go index 48a8245f08..00d4e51536 100644 --- a/providers/cloudns/api.go +++ b/providers/cloudns/api.go @@ -1,23 +1,30 @@ package cloudns import ( + "context" "encoding/json" "fmt" "io" "net/http" "strconv" - "time" + "sync" + + "golang.org/x/time/rate" ) // Api layer for ClouDNS type cloudnsProvider struct { - domainIndex map[string]string - nameserversNames []string - creds struct { + creds struct { id string password string subid string } + + requestLimit *rate.Limiter + + sync.Mutex // Protects all access to the following fields: + domainIndex map[string]string + nameserversNames []string } type requestParams map[string]string @@ -83,76 +90,81 @@ type domainRecord struct { type recordResponse map[string]domainRecord -var allowedTTLValues = []uint32{} +func (c *cloudnsProvider) fetchAvailableNameservers() ([]string, error) { + c.Lock() + defer c.Unlock() -func (c *cloudnsProvider) fetchAvailableNameservers() error { - c.nameserversNames = nil + if c.nameserversNames == nil { - var bodyString, err = c.get("/dns/available-name-servers.json", requestParams{}) - if err != nil { - return fmt.Errorf("failed fetching available nameservers list from ClouDNS: %s", err) - } + var bodyString, err = c.get("/dns/available-name-servers.json", requestParams{}) + if err != nil { + return nil, fmt.Errorf("failed fetching available nameservers list from ClouDNS: %s", err) + } - var nr nameserverResponse - json.Unmarshal(bodyString, &nr) + var nr nameserverResponse + json.Unmarshal(bodyString, &nr) - for _, nameserver := range nr { - if nameserver.Type == "premium" { - c.nameserversNames = append(c.nameserversNames, nameserver.Name) - } + for _, nameserver := range nr { + if nameserver.Type == "premium" { + c.nameserversNames = append(c.nameserversNames, nameserver.Name) + } + } } - return nil + return c.nameserversNames, nil } -func (c *cloudnsProvider) fetchAvailableTTLValues(domain string) error { - allowedTTLValues = nil +func (c *cloudnsProvider) fetchAvailableTTLValues(domain string) ([]uint32, error) { + allowedTTLValues := make([]uint32, 0) params := requestParams{ "domain-name": domain, } var bodyString, err = c.get("/dns/get-available-ttl.json", params) if err != nil { - return fmt.Errorf("failed fetching available TTL values list from ClouDNS: %s", err) + return nil, fmt.Errorf("failed fetching available TTL values list from ClouDNS: %s", err) } json.Unmarshal(bodyString, &allowedTTLValues) - return nil + return allowedTTLValues, nil } -func (c *cloudnsProvider) fetchDomainList() error { - if c.domainIndex != nil { - return nil - } - - rowsPerPage := 100 - page := 1 - for { - var dr zoneResponse - params := requestParams{ - "page": strconv.Itoa(page), - "rows-per-page": strconv.Itoa(rowsPerPage), - } - endpoint := "/dns/list-zones.json" - var bodyString, err = c.get(endpoint, params) - if err != nil { - return fmt.Errorf("failed fetching domain list from ClouDNS: %s", err) - } - json.Unmarshal(bodyString, &dr) +func (c *cloudnsProvider) fetchDomainIndex(name string) (string, bool, error) { + c.Lock() + defer c.Unlock() + + if c.domainIndex == nil { + rowsPerPage := 100 + page := 1 + for { + var dr zoneResponse + params := requestParams{ + "page": strconv.Itoa(page), + "rows-per-page": strconv.Itoa(rowsPerPage), + } + endpoint := "/dns/list-zones.json" + var bodyString, err = c.get(endpoint, params) + if err != nil { + return "", false, fmt.Errorf("failed fetching domain list from ClouDNS: %s", err) + } + json.Unmarshal(bodyString, &dr) - if c.domainIndex == nil { - c.domainIndex = map[string]string{} - } + if c.domainIndex == nil { + c.domainIndex = map[string]string{} + } - for _, domain := range dr { - c.domainIndex[domain.Name] = domain.Name - } - if len(dr) < rowsPerPage { - break + for _, domain := range dr { + c.domainIndex[domain.Name] = domain.Name + } + if len(dr) < rowsPerPage { + break + } + page++ } - page++ } - return nil + + index, ok := c.domainIndex[name] + return index, ok, nil } func (c *cloudnsProvider) createDomain(domain string) error { @@ -267,9 +279,9 @@ func (c *cloudnsProvider) get(endpoint string, params requestParams) ([]byte, er req.URL.RawQuery = q.Encode() // ClouDNS has a rate limit (not documented) of 10 request/second - // so we do a very primitive rate-limiting here - delay every request for 100ms - so max. 10 requests/second ... - time.Sleep(100 * time.Millisecond) + c.requestLimit.Wait(context.Background()) resp, err := client.Do(req) + if err != nil { return []byte{}, err } @@ -290,7 +302,7 @@ func (c *cloudnsProvider) get(endpoint string, params requestParams) ([]byte, er return bodyString, nil } -func fixTTL(ttl uint32) uint32 { +func fixTTL(allowedTTLValues []uint32, ttl uint32) uint32 { // if the TTL is larger than the largest allowed value, return the largest allowed value if ttl > allowedTTLValues[len(allowedTTLValues)-1] { return allowedTTLValues[len(allowedTTLValues)-1] diff --git a/providers/cloudns/cloudnsProvider.go b/providers/cloudns/cloudnsProvider.go index 83aed2eda1..16af5263ae 100644 --- a/providers/cloudns/cloudnsProvider.go +++ b/providers/cloudns/cloudnsProvider.go @@ -10,6 +10,7 @@ import ( "github.com/StackExchange/dnscontrol/v4/pkg/diff" "github.com/StackExchange/dnscontrol/v4/providers" "github.com/miekg/dns/dnsutil" + "golang.org/x/time/rate" ) /* @@ -22,6 +23,7 @@ Info required in `creds.json`: // NewCloudns creates the provider. func NewCloudns(m map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) { c := &cloudnsProvider{} + c.requestLimit = rate.NewLimiter(10, 10) c.creds.id, c.creds.password, c.creds.subid = m["auth-id"], m["auth-password"], m["sub-auth-id"] @@ -37,7 +39,7 @@ var features = providers.DocumentationNotes{ // See providers/capabilities.go for the entire list of capabilities. providers.CanAutoDNSSEC: providers.Can(), providers.CanGetZones: providers.Can(), - providers.CanConcur: providers.Cannot(), + providers.CanConcur: providers.Can(), providers.CanUseAlias: providers.Can(), providers.CanUseCAA: providers.Can(), providers.CanUseDNAME: providers.Can(), @@ -66,10 +68,12 @@ func init() { // GetNameservers returns the nameservers for a domain. func (c *cloudnsProvider) GetNameservers(domain string) ([]*models.Nameserver, error) { - if len(c.nameserversNames) == 0 { - c.fetchAvailableNameservers() + names, err := c.fetchAvailableNameservers() + if err != nil { + return nil, err } - return models.ToNameservers(c.nameserversNames) + + return models.ToNameservers(names) } // // GetDomainCorrections returns the corrections for a domain. @@ -111,23 +115,23 @@ func (c *cloudnsProvider) GetNameservers(domain string) ([]*models.Nameserver, e // GetZoneRecordsCorrections returns a list of corrections that will turn existing records into dc.Records. func (c *cloudnsProvider) GetZoneRecordsCorrections(dc *models.DomainConfig, existingRecords models.Records) ([]*models.Correction, int, error) { - - if c.domainIndex == nil { - if err := c.fetchDomainList(); err != nil { - return nil, 0, err - } - } - domainID, ok := c.domainIndex[dc.Name] - if !ok { + domainID, ok, err := c.fetchDomainIndex(dc.Name) + if err != nil { + return nil, 0, err + } else if !ok { return nil, 0, fmt.Errorf("'%s' not a zone in ClouDNS account", dc.Name) } // Get a list of available TTL values. // The TTL list needs to be obtained for each domain, so get it first here. - c.fetchAvailableTTLValues(dc.Name) + allowedTTLValues, err := c.fetchAvailableTTLValues(dc.Name) + if err != nil { + return nil, 0, err + } + // ClouDNS can only be specified from a specific TTL list, so change the TTL in advance. for _, record := range dc.Records { - record.TTL = fixTTL(record.TTL) + record.TTL = fixTTL(allowedTTLValues, record.TTL) } dnssecFixes, err := c.getDNSSECCorrections(dc) @@ -270,11 +274,9 @@ func (c *cloudnsProvider) GetZoneRecords(domain string, meta map[string]string) // EnsureZoneExists creates a zone if it does not exist func (c *cloudnsProvider) EnsureZoneExists(domain string) error { - if err := c.fetchDomainList(); err != nil { + if _, ok, err := c.fetchDomainIndex(domain); err != nil { return err - } - // zone already exists - if _, ok := c.domainIndex[domain]; ok { + } else if ok { // zone already exists return nil } return c.createDomain(domain)