Merge branch 'main' into tlim_deps

documentation/integration-tests.md

@@ -47,6 +47,14 @@ go test -v -verbose -provider ROUTE53 -end 5
 go test -v -verbose -provider ROUTE53 -start 16 -end 20
 ```
 
+For some providers it may be necessary to increase the test timeout using `-test`. The default is 10 minutes.  `0` is "no limit".  Typical Go durations work too (`1h` for 1 hour, etc).
+
+```shell
+go test -timeout 0 -v -verbose -provider CLOUDNS 
+```
+
+FYI: The order of the flags matters.  Flags native to the Go testing suite (`-timeout` and `-v`) must come before flags that are part of the DNSControl integration tests (`-verbose`, `-provider`). Yeah, that sucks and is confusing.
+
 The actual tests are in the file `integrationTest/integration_test.go`.  The
 tests are in a little language which can be used to describe just about any
 interaction with the API.  Look for the comment `START HERE` or the line

documentation/providers.md

@@ -22,7 +22,7 @@ If a feature is definitively not supported for whatever reason, we would also li
 | [`BIND`](provider/bind.md) | ✅ | ✅ | ❌ | ❌ | ❔ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | [`BUNNY_DNS`](provider/bunny_dns.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❔ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❔ | ❌ | ❌ | ❌ | ❔ | ❔ | ❌ | ✅ | ✅ |
 | [`CLOUDFLAREAPI`](provider/cloudflareapi.md) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | ✅ | ❌ | ✅ | ✅ | ❔ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | ❔ | ❌ | ❌ | ✅ | ✅ |
-| [`CLOUDNS`](provider/cloudns.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❔ | ✅ | ❔ | ✅ | ❔ | ✅ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ✅ | ✅ |
+| [`CLOUDNS`](provider/cloudns.md) | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ | ❔ | ✅ | ❔ | ✅ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ✅ | ✅ |
 | [`CNR`](provider/cnr.md) | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
 | [`CSCGLOBAL`](provider/cscglobal.md) | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❌ | ✅ |
 | [`DESEC`](provider/desec.md) | ❌ | ✅ | ❌ | ✅ | ❔ | ✅ | ✅ | ✅ | ❔ | ✅ | ✅ | ❔ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | ✅ | ❔ | ✅ | ✅ |
@@ -38,7 +38,7 @@ If a feature is definitively not supported for whatever reason, we would also li
 | [`GCLOUD`](provider/gcloud.md) | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | ✅ | ❌ | ❔ | ✅ | ❔ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
 | [`GCORE`](provider/gcore.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❔ | ✅ | ❌ | ✅ | ❌ | ❌ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
 | [`HEDNS`](provider/hedns.md) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
-| [`HETZNER`](provider/hetzner.md) | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❔ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❔ | ✅ | ✅ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
+| [`HETZNER`](provider/hetzner.md) | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❔ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❔ | ✅ | ✅ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
 | [`HEXONET`](provider/hexonet.md) | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ❔ | ✅ | ❔ | ❔ | ✅ | ❔ | ❔ | ❔ | ❔ | ✅ | ✅ | ❔ |
 | [`HOSTINGDE`](provider/hostingde.md) | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❔ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ | ✅ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |
 | [`HUAWEICLOUD`](provider/huaweicloud.md) | ❌ | ✅ | ❌ | ❔ | ❌ | ✅ | ❔ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❔ | ❔ | ❔ | ✅ | ✅ | ✅ |

go.mod

@@ -74,6 +74,7 @@ require (
 	github.com/vultr/govultr/v2 v2.17.2
 	golang.org/x/exp v0.0.0-20241210194714-1829a127f884
 	golang.org/x/text v0.21.0
+	golang.org/x/time v0.8.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -156,7 +157,6 @@ require (
 	golang.org/x/mod v0.22.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/time v0.8.0 // indirect
 	golang.org/x/tools v0.28.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect
 	google.golang.org/grpc v1.67.1 // indirect

providers/cloudns/api.go

@@ -1,23 +1,30 @@
 package cloudns
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"strconv"
-	"time"
+	"sync"
+
+	"golang.org/x/time/rate"
 )
 
 // Api layer for ClouDNS
 type cloudnsProvider struct {
-	domainIndex      map[string]string
-	nameserversNames []string
-	creds            struct {
+	creds struct {
 		id       string
 		password string
 		subid    string
 	}
+
+	requestLimit *rate.Limiter
+
+	sync.Mutex       // Protects all access to the following fields:
+	domainIndex      map[string]string
+	nameserversNames []string
 }
 
 type requestParams map[string]string
@@ -83,76 +90,81 @@ type domainRecord struct {
 
 type recordResponse map[string]domainRecord
 
-var allowedTTLValues = []uint32{}
+func (c *cloudnsProvider) fetchAvailableNameservers() ([]string, error) {
+	c.Lock()
+	defer c.Unlock()
 
-func (c *cloudnsProvider) fetchAvailableNameservers() error {
-	c.nameserversNames = nil
+	if c.nameserversNames == nil {
 
-	var bodyString, err = c.get("/dns/available-name-servers.json", requestParams{})
-	if err != nil {
-		return fmt.Errorf("failed fetching available nameservers list from ClouDNS: %s", err)
-	}
+		var bodyString, err = c.get("/dns/available-name-servers.json", requestParams{})
+		if err != nil {
+			return nil, fmt.Errorf("failed fetching available nameservers list from ClouDNS: %s", err)
+		}
 
-	var nr nameserverResponse
-	json.Unmarshal(bodyString, &nr)
+		var nr nameserverResponse
+		json.Unmarshal(bodyString, &nr)
 
-	for _, nameserver := range nr {
-		if nameserver.Type == "premium" {
-			c.nameserversNames = append(c.nameserversNames, nameserver.Name)
-		}
+		for _, nameserver := range nr {
+			if nameserver.Type == "premium" {
+				c.nameserversNames = append(c.nameserversNames, nameserver.Name)
+			}
 
+		}
 	}
-	return nil
+	return c.nameserversNames, nil
 }
 
-func (c *cloudnsProvider) fetchAvailableTTLValues(domain string) error {
-	allowedTTLValues = nil
+func (c *cloudnsProvider) fetchAvailableTTLValues(domain string) ([]uint32, error) {
+	allowedTTLValues := make([]uint32, 0)
 	params := requestParams{
 		"domain-name": domain,
 	}
 
 	var bodyString, err = c.get("/dns/get-available-ttl.json", params)
 	if err != nil {
-		return fmt.Errorf("failed fetching available TTL values list from ClouDNS: %s", err)
+		return nil, fmt.Errorf("failed fetching available TTL values list from ClouDNS: %s", err)
 	}
 
 	json.Unmarshal(bodyString, &allowedTTLValues)
-	return nil
+	return allowedTTLValues, nil
 }
 
-func (c *cloudnsProvider) fetchDomainList() error {
-	if c.domainIndex != nil {
-		return nil
-	}
-
-	rowsPerPage := 100
-	page := 1
-	for {
-		var dr zoneResponse
-		params := requestParams{
-			"page":          strconv.Itoa(page),
-			"rows-per-page": strconv.Itoa(rowsPerPage),
-		}
-		endpoint := "/dns/list-zones.json"
-		var bodyString, err = c.get(endpoint, params)
-		if err != nil {
-			return fmt.Errorf("failed fetching domain list from ClouDNS: %s", err)
-		}
-		json.Unmarshal(bodyString, &dr)
+func (c *cloudnsProvider) fetchDomainIndex(name string) (string, bool, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.domainIndex == nil {
+		rowsPerPage := 100
+		page := 1
+		for {
+			var dr zoneResponse
+			params := requestParams{
+				"page":          strconv.Itoa(page),
+				"rows-per-page": strconv.Itoa(rowsPerPage),
+			}
+			endpoint := "/dns/list-zones.json"
+			var bodyString, err = c.get(endpoint, params)
+			if err != nil {
+				return "", false, fmt.Errorf("failed fetching domain list from ClouDNS: %s", err)
+			}
+			json.Unmarshal(bodyString, &dr)
 
-		if c.domainIndex == nil {
-			c.domainIndex = map[string]string{}
-		}
+			if c.domainIndex == nil {
+				c.domainIndex = map[string]string{}
+			}
 
-		for _, domain := range dr {
-			c.domainIndex[domain.Name] = domain.Name
-		}
-		if len(dr) < rowsPerPage {
-			break
+			for _, domain := range dr {
+				c.domainIndex[domain.Name] = domain.Name
+			}
+			if len(dr) < rowsPerPage {
+				break
+			}
+			page++
 		}
-		page++
 	}
-	return nil
+
+	index, ok := c.domainIndex[name]
+	return index, ok, nil
 }
 
 func (c *cloudnsProvider) createDomain(domain string) error {
@@ -267,9 +279,9 @@ func (c *cloudnsProvider) get(endpoint string, params requestParams) ([]byte, er
 	req.URL.RawQuery = q.Encode()
 
 	// ClouDNS has a rate limit (not documented) of 10 request/second
-	// so we do a very primitive rate-limiting here - delay every request for 100ms - so max. 10 requests/second ...
-	time.Sleep(100 * time.Millisecond)
+	c.requestLimit.Wait(context.Background())
 	resp, err := client.Do(req)
+
 	if err != nil {
 		return []byte{}, err
 	}
@@ -290,7 +302,7 @@ func (c *cloudnsProvider) get(endpoint string, params requestParams) ([]byte, er
 	return bodyString, nil
 }
 
-func fixTTL(ttl uint32) uint32 {
+func fixTTL(allowedTTLValues []uint32, ttl uint32) uint32 {
 	// if the TTL is larger than the largest allowed value, return the largest allowed value
 	if ttl > allowedTTLValues[len(allowedTTLValues)-1] {
 		return allowedTTLValues[len(allowedTTLValues)-1]

providers/cloudns/cloudnsProvider.go

@@ -10,6 +10,7 @@ import (
 	"github.com/StackExchange/dnscontrol/v4/pkg/diff"
 	"github.com/StackExchange/dnscontrol/v4/providers"
 	"github.com/miekg/dns/dnsutil"
+	"golang.org/x/time/rate"
 )
 
 /*
@@ -22,6 +23,7 @@ Info required in `creds.json`:
 // NewCloudns creates the provider.
 func NewCloudns(m map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
 	c := &cloudnsProvider{}
+	c.requestLimit = rate.NewLimiter(10, 10)
 
 	c.creds.id, c.creds.password, c.creds.subid = m["auth-id"], m["auth-password"], m["sub-auth-id"]
 
@@ -37,7 +39,7 @@ var features = providers.DocumentationNotes{
 	// See providers/capabilities.go for the entire list of capabilities.
 	providers.CanAutoDNSSEC:          providers.Can(),
 	providers.CanGetZones:            providers.Can(),
-	providers.CanConcur:              providers.Cannot(),
+	providers.CanConcur:              providers.Can(),
 	providers.CanUseAlias:            providers.Can(),
 	providers.CanUseCAA:              providers.Can(),
 	providers.CanUseDNAME:            providers.Can(),
@@ -66,10 +68,12 @@ func init() {
 
 // GetNameservers returns the nameservers for a domain.
 func (c *cloudnsProvider) GetNameservers(domain string) ([]*models.Nameserver, error) {
-	if len(c.nameserversNames) == 0 {
-		c.fetchAvailableNameservers()
+	names, err := c.fetchAvailableNameservers()
+	if err != nil {
+		return nil, err
 	}
-	return models.ToNameservers(c.nameserversNames)
+
+	return models.ToNameservers(names)
 }
 
 // // GetDomainCorrections returns the corrections for a domain.
@@ -111,23 +115,23 @@ func (c *cloudnsProvider) GetNameservers(domain string) ([]*models.Nameserver, e
 
 // GetZoneRecordsCorrections returns a list of corrections that will turn existing records into dc.Records.
 func (c *cloudnsProvider) GetZoneRecordsCorrections(dc *models.DomainConfig, existingRecords models.Records) ([]*models.Correction, int, error) {
-
-	if c.domainIndex == nil {
-		if err := c.fetchDomainList(); err != nil {
-			return nil, 0, err
-		}
-	}
-	domainID, ok := c.domainIndex[dc.Name]
-	if !ok {
+	domainID, ok, err := c.fetchDomainIndex(dc.Name)
+	if err != nil {
+		return nil, 0, err
+	} else if !ok {
 		return nil, 0, fmt.Errorf("'%s' not a zone in ClouDNS account", dc.Name)
 	}
 
 	// Get a list of available TTL values.
 	// The TTL list needs to be obtained for each domain, so get it first here.
-	c.fetchAvailableTTLValues(dc.Name)
+	allowedTTLValues, err := c.fetchAvailableTTLValues(dc.Name)
+	if err != nil {
+		return nil, 0, err
+	}
+
 	// ClouDNS can only be specified from a specific TTL list, so change the TTL in advance.
 	for _, record := range dc.Records {
-		record.TTL = fixTTL(record.TTL)
+		record.TTL = fixTTL(allowedTTLValues, record.TTL)
 	}
 
 	dnssecFixes, err := c.getDNSSECCorrections(dc)
@@ -270,11 +274,9 @@ func (c *cloudnsProvider) GetZoneRecords(domain string, meta map[string]string)
 
 // EnsureZoneExists creates a zone if it does not exist
 func (c *cloudnsProvider) EnsureZoneExists(domain string) error {
-	if err := c.fetchDomainList(); err != nil {
+	if _, ok, err := c.fetchDomainIndex(domain); err != nil {
 		return err
-	}
-	// zone already exists
-	if _, ok := c.domainIndex[domain]; ok {
+	} else if ok { // zone already exists
 		return nil
 	}
 	return c.createDomain(domain)