From 350c8654929acea83c9b6c82111d2ad3fa32b734 Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Sat, 23 Sep 2023 14:38:20 -0400 Subject: [PATCH 1/7] silence --- documentation/functions/domain/IGNORE.md | 11 ++++-- models/record.go | 1 + models/unmanaged.go | 3 ++ pkg/diff2/handsoff.go | 35 +++++++++++++------ pkg/js/helpers.js | 6 +++- pkg/js/parse_tests/005-ignored-records.js | 2 ++ pkg/js/parse_tests/005-ignored-records.json | 37 ++++++++++++++++----- 7 files changed, 71 insertions(+), 24 deletions(-) diff --git a/documentation/functions/domain/IGNORE.md b/documentation/functions/domain/IGNORE.md index e840a904c2..5c48d4cb06 100644 --- a/documentation/functions/domain/IGNORE.md +++ b/documentation/functions/domain/IGNORE.md @@ -4,10 +4,12 @@ parameters: - labelSpec - typeSpec - targetSpec + - silent parameter_types: labelSpec: string typeSpec: string? targetSpec: string? + silent: bool? --- `IGNORE()` makes it possible for DNSControl to share management of a domain @@ -24,9 +26,9 @@ To solve this problem simply include `IGNORE()` statements that identify which records are managed elsewhere. DNSControl will not modify or delete those records. -Technically `IGNORE_NAME` is a promise that DNSControl will not modify or -delete existing records that match particular patterns. It is like -[`NO_PURGE`](../domain/NO_PURGE.md) that matches only specific records. +Technically `IGNORE` is a promise that DNSControl will not modify or +delete existing records that match particular patterns. It is similar to +[`NO_PURGE`](../domain/NO_PURGE.md) but it matches only specific records. Including a record that is ignored is considered an error and may have undefined behavior. This safety check can be disabled using the @@ -38,6 +40,7 @@ The `IGNORE()` function can be used with up to 3 parameters: {% code %} ```javascript +IGNORE(labelSpec, typeSpec, targetSpec, silent): IGNORE(labelSpec, typeSpec, targetSpec): IGNORE(labelSpec, typeSpec): IGNORE(labelSpec): @@ -47,6 +50,7 @@ IGNORE(labelSpec): * `labelSpec` is a glob that matches the DNS label. For example `"foo"` or `"foo*"`. `"*"` matches all labels, as does the empty string (`""`). * `typeSpec` is a comma-separated list of DNS types. For example `"A"` matches DNS A records, `"A,CNAME"` matches both A and CNAME records. `"*"` matches any DNS type, as does the empty string (`""`). * `targetSpec` is a glob that matches the DNS target. For example `"foo"` or `"foo*"`. `"*"` matches all targets, as does the empty string (`""`). +* `silent` is a bool that, when set to `true`, indicates that "ignored record reports" do not need to mention these records. Use `--full` to list all records being ignored. Note that if two or more `IGNORE()` statements both match the same record and have different `silent` settings, it is non-deterministic whether or not the report will include this record. `typeSpec` and `targetSpec` default to `"*"` if they are omitted. @@ -78,6 +82,7 @@ D("example.com", REG_MY_PROVIDER, DnsProvider(DSP_MY_PROVIDER), IGNORE("bar", "A,MX"), // ignore only A and MX records for name bar IGNORE("*", "*", "dev-*"), // Ignore targets with a `dev-` prefix IGNORE("*", "A", "1\.2\.3\."), // Ignore targets in the 1.2.3.0/24 CIDR block + IGNORE("*", "A", "1\.2\.3\."), // Same as previous, but records will not be reported unless `--full` END); ``` {% endcode %} diff --git a/models/record.go b/models/record.go index 894c49dace..635b9f9bed 100644 --- a/models/record.go +++ b/models/record.go @@ -130,6 +130,7 @@ type RecordConfig struct { TxtStrings []string `json:"txtstrings,omitempty"` // TxtStrings stores all strings (including the first). Target stores all the strings joined. R53Alias map[string]string `json:"r53_alias,omitempty"` AzureAlias map[string]string `json:"azure_alias,omitempty"` + SilenceReporting bool `json:"silence_reporting,omitempty"` } // MarshalJSON marshals RecordConfig. diff --git a/models/unmanaged.go b/models/unmanaged.go index e1f7f5d7e7..9e64d8b3fb 100644 --- a/models/unmanaged.go +++ b/models/unmanaged.go @@ -26,6 +26,9 @@ type UnmanagedConfig struct { // Glob pattern for matching targets. TargetPattern string `json:"target_pattern,omitempty"` TargetGlob glob.Glob `json:"-"` // Compiled version + + // Output warnings about this ignore? + SilenceReporting bool `json:"silence_reporting,omitempty"` } // DebugUnmanagedConfig returns a string version of an []*UnmanagedConfig for debugging purposes. diff --git a/pkg/diff2/handsoff.go b/pkg/diff2/handsoff.go index 917c33af7b..bfdf3b3b3f 100644 --- a/pkg/diff2/handsoff.go +++ b/pkg/diff2/handsoff.go @@ -123,14 +123,14 @@ func handsoff( msgs = append(msgs, reportSkips(foreign, !printer.SkinnyReport)...) } if len(ignorable) != 0 { - msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of IGNORE*():", len(ignorable))) + msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of IGNORE():", len(ignorable))) msgs = append(msgs, reportSkips(ignorable, !printer.SkinnyReport)...) } // Check for invalid use of IGNORE_*. conflicts := findConflicts(unmanagedConfigs, desired) if len(conflicts) != 0 { - msgs = append(msgs, fmt.Sprintf("%d records that are both IGNORE*()'d and not ignored:", len(conflicts))) + msgs = append(msgs, fmt.Sprintf("%d records that are both IGNORE()'d and not ignored:", len(conflicts))) for _, r := range conflicts { msgs = append(msgs, fmt.Sprintf(" %s %s %s", r.GetLabelFQDN(), r.Type, r.GetTargetCombined())) } @@ -148,15 +148,27 @@ func handsoff( // reportSkips reports records being skipped, if !full only the first maxReport are output. func reportSkips(recs models.Records, full bool) []string { - var msgs []string - shorten := (!full) && (len(recs) > maxReport) - last := len(recs) + var prints []*models.RecordConfig + if full { + prints = recs + } else { + for i := range recs { + fmt.Printf("DEBUG: silence=%v rec=%v\n", recs[i].SilenceReporting, *recs[i]) + if !recs[i].SilenceReporting { + prints = append(prints, recs[i]) + } + } + } + + shorten := (len(recs) > maxReport) + last := len(prints) if shorten { last = maxReport } - for _, r := range recs[:last] { + var msgs []string + for _, r := range prints[:last] { msgs = append(msgs, fmt.Sprintf(" %s. %s %s", r.GetLabelFQDN(), r.Type, r.GetTargetCombined())) } if shorten { @@ -173,9 +185,10 @@ func processIgnoreAndNoPurge(domain string, existing, desired, absences models.R absentDB := models.NewRecordDBFromRecords(absences, domain) compileUnmanagedConfigs(unmanagedConfigs) for _, rec := range existing { - isMatch := matchAny(unmanagedConfigs, rec) + isMatch, silence := matchAny(unmanagedConfigs, rec) //fmt.Printf("DEBUG: matchAny returned: %v\n", isMatch) if isMatch { + rec.SilenceReporting = silence ignorable = append(ignorable, rec) } else { if noPurge { @@ -197,7 +210,7 @@ func processIgnoreAndNoPurge(domain string, existing, desired, absences models.R func findConflicts(uconfigs []*models.UnmanagedConfig, recs models.Records) models.Records { var conflicts models.Records for _, rec := range recs { - if matchAny(uconfigs, rec) { + if ans, _ := matchAny(uconfigs, rec); ans { conflicts = append(conflicts, rec) } } @@ -241,16 +254,16 @@ func compileUnmanagedConfigs(configs []*models.UnmanagedConfig) error { } // matchAny returns true if rec matches any of the uconfigs. -func matchAny(uconfigs []*models.UnmanagedConfig, rec *models.RecordConfig) bool { +func matchAny(uconfigs []*models.UnmanagedConfig, rec *models.RecordConfig) (bool, bool) { //fmt.Printf("DEBUG: matchAny(%s, %q, %q, %q)\n", models.DebugUnmanagedConfig(uconfigs), rec.NameFQDN, rec.Type, rec.GetTargetField()) for _, uc := range uconfigs { if matchLabel(uc.LabelGlob, rec.GetLabel()) && matchType(uc.RTypeMap, rec.Type) && matchTarget(uc.TargetGlob, rec.GetTargetField()) { - return true + return true, uc.SilenceReporting } } - return false + return false, false } func matchLabel(labelGlob glob.Glob, labelName string) bool { if labelGlob == nil { diff --git a/pkg/js/helpers.js b/pkg/js/helpers.js index 10dfa1f694..8e5c36d20a 100644 --- a/pkg/js/helpers.js +++ b/pkg/js/helpers.js @@ -842,7 +842,7 @@ var IGNORE_NAME_DISABLE_SAFETY_CHECK = { }; // IGNORE(labelPattern, rtypePattern, targetPattern) -function IGNORE(labelPattern, rtypePattern, targetPattern) { +function IGNORE(labelPattern, rtypePattern, targetPattern, silently) { if (labelPattern === undefined) { labelPattern = '*'; } @@ -852,6 +852,9 @@ function IGNORE(labelPattern, rtypePattern, targetPattern) { if (targetPattern === undefined) { targetPattern = '*'; } + if (silently === undefined) { + silently = false; + } return function (d) { // diff1 d.ignored_names.push({ pattern: labelPattern, types: rtypePattern }); @@ -860,6 +863,7 @@ function IGNORE(labelPattern, rtypePattern, targetPattern) { label_pattern: labelPattern, rType_pattern: rtypePattern, target_pattern: targetPattern, + silence_reporting: silently, }); }; } diff --git a/pkg/js/parse_tests/005-ignored-records.js b/pkg/js/parse_tests/005-ignored-records.js index f389c92d69..25693dc95d 100644 --- a/pkg/js/parse_tests/005-ignored-records.js +++ b/pkg/js/parse_tests/005-ignored-records.js @@ -16,6 +16,8 @@ D("diff2.com", "none" , IGNORE("", "A,AAAA") , IGNORE("", "", "mytarget") , IGNORE("labelc", "CNAME", "targetc") + , IGNORE("silenttrue", "CNAME", "targetc", true) + , IGNORE("silentfalse", "CNAME", "targetc", false) // Compatibility mode: , IGNORE_NAME("nametest") , IGNORE_TARGET("targettest1") diff --git a/pkg/js/parse_tests/005-ignored-records.json b/pkg/js/parse_tests/005-ignored-records.json index 0a9f748387..43db77f312 100644 --- a/pkg/js/parse_tests/005-ignored-records.json +++ b/pkg/js/parse_tests/005-ignored-records.json @@ -1,12 +1,8 @@ { - "registrars": [], "dns_providers": [], "domains": [ { - "name": "foo.com", - "registrar": "none", "dnsProviders": {}, - "records": [], "ignored_names": [ { "pattern": "testignore", @@ -43,6 +39,9 @@ "type": "CNAME" } ], + "name": "foo.com", + "records": [], + "registrar": "none", "unmanaged": [ { "label_pattern": "testignore", @@ -80,10 +79,7 @@ ] }, { - "name": "diff2.com", - "registrar": "none", "dnsProviders": {}, - "records": [], "ignored_names": [ { "pattern": "mylabel", @@ -113,6 +109,14 @@ "pattern": "labelc", "types": "CNAME" }, + { + "pattern": "silenttrue", + "types": "CNAME" + }, + { + "pattern": "silentfalse", + "types": "CNAME" + }, { "pattern": "nametest", "types": "*" @@ -128,6 +132,9 @@ "type": "A" } ], + "name": "diff2.com", + "records": [], + "registrar": "none", "unmanaged": [ { "label_pattern": "mylabel", @@ -157,6 +164,17 @@ "rType_pattern": "CNAME", "target_pattern": "targetc" }, + { + "label_pattern": "silenttrue", + "rType_pattern": "CNAME", + "silence_reporting": true, + "target_pattern": "targetc" + }, + { + "label_pattern": "silentfalse", + "rType_pattern": "CNAME", + "target_pattern": "targetc" + }, { "label_pattern": "nametest", "rType_pattern": "*" @@ -170,5 +188,6 @@ } ] } - ] -} \ No newline at end of file + ], + "registrars": [] +} From 5e20497e0d2f7111737d03cbc06fd49de003c1e7 Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Sun, 24 Sep 2023 10:11:22 -0400 Subject: [PATCH 2/7] reportSkips works --- pkg/diff2/handsoff.go | 50 +++++++++++++++++++------------------- pkg/diff2/handsoff_test.go | 35 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 25 deletions(-) diff --git a/pkg/diff2/handsoff.go b/pkg/diff2/handsoff.go index bfdf3b3b3f..214365fb3d 100644 --- a/pkg/diff2/handsoff.go +++ b/pkg/diff2/handsoff.go @@ -98,7 +98,7 @@ The actual implementation combines this all into one loop: Append "foreign list" to "desired". */ -const maxReport = 5 +const defaultMaxReport = 5 // handsoff processes the IGNORE*()//NO_PURGE/ENSURE_ABSENT features. func handsoff( @@ -120,11 +120,11 @@ func handsoff( ignorable, foreign := processIgnoreAndNoPurge(domain, existing, desired, absences, unmanagedConfigs, noPurge) if len(foreign) != 0 { msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of NO_PURGE:", len(foreign))) - msgs = append(msgs, reportSkips(foreign, !printer.SkinnyReport)...) + msgs = append(msgs, reportSkips(foreign, defaultMaxReport, !printer.SkinnyReport)...) } if len(ignorable) != 0 { - msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of IGNORE():", len(ignorable))) - msgs = append(msgs, reportSkips(ignorable, !printer.SkinnyReport)...) + msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of IGNORE()", len(ignorable))) + msgs = append(msgs, reportSkips(ignorable, defaultMaxReport, !printer.SkinnyReport)...) } // Check for invalid use of IGNORE_*. @@ -147,37 +147,37 @@ func handsoff( } // reportSkips reports records being skipped, if !full only the first maxReport are output. -func reportSkips(recs models.Records, full bool) []string { +func reportSkips(recs models.Records, maxReport int, full bool) (msgs []string) { + if len(recs) == 0 { + return nil + } - var prints []*models.RecordConfig if full { - prints = recs - } else { - for i := range recs { - fmt.Printf("DEBUG: silence=%v rec=%v\n", recs[i].SilenceReporting, *recs[i]) - if !recs[i].SilenceReporting { - prints = append(prints, recs[i]) - } + for _, r := range recs { + msgs = append(msgs, genLine(r)) } + return msgs } - shorten := (len(recs) > maxReport) - last := len(prints) - if shorten { - last = maxReport - } - - var msgs []string - for _, r := range prints[:last] { - msgs = append(msgs, fmt.Sprintf(" %s. %s %s", r.GetLabelFQDN(), r.Type, r.GetTargetCombined())) + for _, r := range recs { + //fmt.Printf("DEBUG: silence=%v rec=%v\n", recs[i].SilenceReporting, *recs[i]) + if !r.SilenceReporting { + msgs = append(msgs, genLine(r)) + if len(msgs) == maxReport { + break + } + } } - if shorten { - msgs = append(msgs, fmt.Sprintf(" ...and %d more... (use --full to show all)", len(recs)-maxReport)) + if len(msgs) < len(recs) { + msgs = append(msgs, fmt.Sprintf(" ...%d records not displayed.", len(recs)-len(msgs))) } - return msgs } +func genLine(r *models.RecordConfig) string { + return fmt.Sprintf(" %s. %s %s", r.GetLabelFQDN(), r.Type, r.GetTargetCombined()) +} + // processIgnoreAndNoPurge processes the IGNORE_*() and NO_PURGE/ENSURE_ABSENT() features. func processIgnoreAndNoPurge(domain string, existing, desired, absences models.Records, unmanagedConfigs []*models.UnmanagedConfig, noPurge bool) (models.Records, models.Records) { var ignorable, foreign models.Records diff --git a/pkg/diff2/handsoff_test.go b/pkg/diff2/handsoff_test.go index 94e315292a..2d3c4a0ce6 100644 --- a/pkg/diff2/handsoff_test.go +++ b/pkg/diff2/handsoff_test.go @@ -234,3 +234,38 @@ _2222222222222222.cr CNAME _333333.nnn.acm-validations.aws. FOREIGN: `) } + +func Test_reportSkips(t *testing.T) { + + yes := &models.RecordConfig{Type: "A", SilenceReporting: true} + no := &models.RecordConfig{Type: "A", SilenceReporting: false} + maxReport := 5 + + tests := []struct { + name string + data models.Records + countNotFull int + }{ + {"no2", models.Records{no, no}, 2}, + {"yes2", models.Records{yes, yes}, 0 + 1}, + {"noMax", models.Records{no, no, no, no, no}, maxReport}, + {"no6", models.Records{no, no, no, no, no, no}, maxReport + 1}, + {"noMaxyes2", models.Records{no, no, no, no, no, yes, yes}, maxReport + 1}, + {"no6yes2", models.Records{no, no, no, no, no, no, yes, yes}, maxReport + 1}, + {"no6yes2mixed", models.Records{yes, no, no, no, yes, no, no, no}, maxReport + 1}, + {"yes8", models.Records{yes, yes, yes, yes, yes, yes, yes, yes}, 0 + 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // If full==true, there should be 1 msg for each item in tt.data: + if got := reportSkips(tt.data, maxReport, true); len(got) != len(tt.data) { + t.Errorf("reportSkips(~, %d, true) = %v, want %v", len(got), maxReport, len(tt.data)) + } + // If full==false, there should be 1 msg for the first maxReport no's: + if got := reportSkips(tt.data, maxReport, false); len(got) != tt.countNotFull { + fmt.Printf("MSGS=%v\n", strings.Join(got, ":\n")) + t.Errorf("reportSkips(-, false) = %v, want %v", len(got), tt.countNotFull) + } + }) + } +} From 439d2f3f5ad83d0e10bc1c1a8a38e60f2bce389a Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Sun, 24 Sep 2023 11:22:22 -0400 Subject: [PATCH 3/7] Pretty up the reporting about IGNORE/NO_PURGE --- pkg/diff2/handsoff.go | 88 +++++++++++++++++++++++++++++++------- pkg/diff2/handsoff_test.go | 16 +++---- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/pkg/diff2/handsoff.go b/pkg/diff2/handsoff.go index 214365fb3d..e6298b4d0c 100644 --- a/pkg/diff2/handsoff.go +++ b/pkg/diff2/handsoff.go @@ -118,14 +118,8 @@ func handsoff( // Process IGNORE*() and NO_PURGE features: ignorable, foreign := processIgnoreAndNoPurge(domain, existing, desired, absences, unmanagedConfigs, noPurge) - if len(foreign) != 0 { - msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of NO_PURGE:", len(foreign))) - msgs = append(msgs, reportSkips(foreign, defaultMaxReport, !printer.SkinnyReport)...) - } - if len(ignorable) != 0 { - msgs = append(msgs, fmt.Sprintf("%d records not being deleted because of IGNORE()", len(ignorable))) - msgs = append(msgs, reportSkips(ignorable, defaultMaxReport, !printer.SkinnyReport)...) - } + msgs = append(msgs, genReport(foreign, "NO_PURGE", defaultMaxReport)...) + msgs = append(msgs, genReport(ignorable, "IGNORE()", defaultMaxReport)...) // Check for invalid use of IGNORE_*. conflicts := findConflicts(unmanagedConfigs, desired) @@ -146,15 +140,78 @@ func handsoff( return desired, msgs, nil } -// reportSkips reports records being skipped, if !full only the first maxReport are output. -func reportSkips(recs models.Records, maxReport int, full bool) (msgs []string) { +// genReport generates a report of what records were not deleted with a human-readable header and footer. Abides by maxReport. +func genReport(recs models.Records, reason string, maxReport int) (msgs []string) { + if len(recs) == 0 { + return nil + } + visibleCount, hiddenCount := countVisibility(recs) + header, footer := makeHeaderFooter(reason, !printer.SkinnyReport, maxReport, len(recs), visibleCount, hiddenCount) + msgs = append(msgs, header) + msgs = append(msgs, reportMessages(recs, maxReport, !printer.SkinnyReport)...) + if footer != "" { + msgs = append(msgs, footer) + } + + return msgs +} + +// makeHeaderFooter generates fancy header and footer. +func makeHeaderFooter(reason string, full bool, maxReport, recsCount, visibleCount, hiddenCount int) (header, footer string) { + + if full { + // No maximum. Everything is shown. + header = fmt.Sprintf("%d records not deleted because of %s:", recsCount, reason) + footer = "" + + } else if visibleCount > maxReport { + // We hit the maxReport limit: + if hiddenCount > 0 { + // Some were hidden intentionally. + header = fmt.Sprintf("%d records not deleted because of %s:", recsCount, reason) + footer = fmt.Sprintf(" ...plus %d others (use --full to reveal)", recsCount-maxReport) + } else { + // Nothing hidden. + header = fmt.Sprintf("%d records not being deleted because of %s:", recsCount, reason) + footer = fmt.Sprintf(" ...%d records not displayed (use --full to show all)", recsCount-maxReport) + } + + // At this point we know that the number of items being reported is less than max. + } else if visibleCount == 0 && hiddenCount != 0 { // Everything is hidden + header = fmt.Sprintf("%d records not being deleted because of %s. (Add --full to reveal)", recsCount, reason) + footer = "" + } else if hiddenCount != 0 { // Some things are hidden + header = fmt.Sprintf("%d records not being deleted because of %s:", recsCount, reason) + footer = fmt.Sprintf(" ...and %d others (use --full to reveal)", hiddenCount) + } else { // Nothing hidden + header = fmt.Sprintf("%d records not being deleted because of %s:", recsCount, reason) + footer = "" + } + + return header, footer +} + +// countVisibility returns how many records are visible/hidden. +func countVisibility(recs models.Records) (visibleCount, hiddenCount int) { + for _, r := range recs { + if r.SilenceReporting { + hiddenCount++ + } else { + visibleCount++ + } + } + return visibleCount, hiddenCount +} + +// reportMessages generates one message for each record, abiding by maxReport limits. +func reportMessages(recs models.Records, maxReport int, full bool) (msgs []string) { if len(recs) == 0 { return nil } if full { for _, r := range recs { - msgs = append(msgs, genLine(r)) + msgs = append(msgs, genRecordMessage(r)) } return msgs } @@ -162,19 +219,18 @@ func reportSkips(recs models.Records, maxReport int, full bool) (msgs []string) for _, r := range recs { //fmt.Printf("DEBUG: silence=%v rec=%v\n", recs[i].SilenceReporting, *recs[i]) if !r.SilenceReporting { - msgs = append(msgs, genLine(r)) + msgs = append(msgs, genRecordMessage(r)) if len(msgs) == maxReport { break } } } - if len(msgs) < len(recs) { - msgs = append(msgs, fmt.Sprintf(" ...%d records not displayed.", len(recs)-len(msgs))) - } + return msgs } -func genLine(r *models.RecordConfig) string { +// genRecordMessage generate the message for one record. +func genRecordMessage(r *models.RecordConfig) string { return fmt.Sprintf(" %s. %s %s", r.GetLabelFQDN(), r.Type, r.GetTargetCombined()) } diff --git a/pkg/diff2/handsoff_test.go b/pkg/diff2/handsoff_test.go index 2d3c4a0ce6..ce2161cd1c 100644 --- a/pkg/diff2/handsoff_test.go +++ b/pkg/diff2/handsoff_test.go @@ -247,22 +247,22 @@ func Test_reportSkips(t *testing.T) { countNotFull int }{ {"no2", models.Records{no, no}, 2}, - {"yes2", models.Records{yes, yes}, 0 + 1}, + {"yes2", models.Records{yes, yes}, 0}, {"noMax", models.Records{no, no, no, no, no}, maxReport}, - {"no6", models.Records{no, no, no, no, no, no}, maxReport + 1}, - {"noMaxyes2", models.Records{no, no, no, no, no, yes, yes}, maxReport + 1}, - {"no6yes2", models.Records{no, no, no, no, no, no, yes, yes}, maxReport + 1}, - {"no6yes2mixed", models.Records{yes, no, no, no, yes, no, no, no}, maxReport + 1}, - {"yes8", models.Records{yes, yes, yes, yes, yes, yes, yes, yes}, 0 + 1}, + {"no6", models.Records{no, no, no, no, no, no}, maxReport}, + {"noMaxyes2", models.Records{no, no, no, no, no, yes, yes}, maxReport}, + {"no6yes2", models.Records{no, no, no, no, no, no, yes, yes}, maxReport}, + {"no6yes2mixed", models.Records{yes, no, no, no, yes, no, no, no}, maxReport}, + {"yes8", models.Records{yes, yes, yes, yes, yes, yes, yes, yes}, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // If full==true, there should be 1 msg for each item in tt.data: - if got := reportSkips(tt.data, maxReport, true); len(got) != len(tt.data) { + if got := reportMessages(tt.data, maxReport, true); len(got) != len(tt.data) { t.Errorf("reportSkips(~, %d, true) = %v, want %v", len(got), maxReport, len(tt.data)) } // If full==false, there should be 1 msg for the first maxReport no's: - if got := reportSkips(tt.data, maxReport, false); len(got) != tt.countNotFull { + if got := reportMessages(tt.data, maxReport, false); len(got) != tt.countNotFull { fmt.Printf("MSGS=%v\n", strings.Join(got, ":\n")) t.Errorf("reportSkips(-, false) = %v, want %v", len(got), tt.countNotFull) } From 22a7e0b155eaad608f69248e519e25220b8b4555 Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Mon, 25 Sep 2023 17:59:36 -0400 Subject: [PATCH 4/7] more simple headers --- documentation/functions/domain/IGNORE.md | 32 +++++++----- pkg/diff2/handsoff.go | 57 +++++++++++---------- pkg/js/parse_tests/005-ignored-records.js | 1 + pkg/js/parse_tests/005-ignored-records.json | 8 +++ 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/documentation/functions/domain/IGNORE.md b/documentation/functions/domain/IGNORE.md index 5c48d4cb06..101b511886 100644 --- a/documentation/functions/domain/IGNORE.md +++ b/documentation/functions/domain/IGNORE.md @@ -14,7 +14,7 @@ parameter_types: `IGNORE()` makes it possible for DNSControl to share management of a domain with an external system. The parameters of `IGNORE()` indicate which records -are managed elsewhere and should not be modified or deleted. +are managed elsewhere and should not be modified or deleted by DNSControl. Use case: Suppose a domain is managed by both DNSControl and a third-party system. This creates a problem because DNSControl will try to delete records @@ -23,20 +23,21 @@ those records. The two systems will get into an endless update cycle where each will revert changes made by the other in an endless loop. To solve this problem simply include `IGNORE()` statements that identify which -records are managed elsewhere. DNSControl will not modify or delete those +records (or patterns) are managed elsewhere. DNSControl will not modify or delete those records. Technically `IGNORE` is a promise that DNSControl will not modify or delete existing records that match particular patterns. It is similar to [`NO_PURGE`](../domain/NO_PURGE.md) but it matches only specific records. -Including a record that is ignored is considered an error and may have -undefined behavior. This safety check can be disabled using the -[`DISABLE_IGNORE_SAFETY_CHECK`](../domain/DISABLE_IGNORE_SAFETY_CHECK.md) feature. +It is considered to be an error if an`IGNORE()` pattern matches records in +`dnsconfig.js`. This safety check can be disabled using the +[`DISABLE_IGNORE_SAFETY_CHECK`](../domain/DISABLE_IGNORE_SAFETY_CHECK.md) +feature. Behavior is undefined when the safety check is disabled. ## Syntax -The `IGNORE()` function can be used with up to 3 parameters: +The `IGNORE()` function can be used with up to 4 parameters: {% code %} ```javascript @@ -48,9 +49,9 @@ IGNORE(labelSpec): {% endcode %} * `labelSpec` is a glob that matches the DNS label. For example `"foo"` or `"foo*"`. `"*"` matches all labels, as does the empty string (`""`). -* `typeSpec` is a comma-separated list of DNS types. For example `"A"` matches DNS A records, `"A,CNAME"` matches both A and CNAME records. `"*"` matches any DNS type, as does the empty string (`""`). +* `typeSpec` is a comma-separated list of DNS types. For example `"A"` matches DNS A records, `"A,CNAME"` matches both A and CNAME records. `"*"` matches any DNS type, as does the empty string (`""`). * `targetSpec` is a glob that matches the DNS target. For example `"foo"` or `"foo*"`. `"*"` matches all targets, as does the empty string (`""`). -* `silent` is a bool that, when set to `true`, indicates that "ignored record reports" do not need to mention these records. Use `--full` to list all records being ignored. Note that if two or more `IGNORE()` statements both match the same record and have different `silent` settings, it is non-deterministic whether or not the report will include this record. +* `silent` is a bool that, when set to `true`, indicates that records ignored using this rule do not need to be listed in `preview` and `push` when they announce ignored records. Use the `--full` flag to show all records. If two or more `IGNORE()` statements match the same record and have different `silent` settings, results are undefined. `typeSpec` and `targetSpec` default to `"*"` if they are omitted. @@ -68,10 +69,14 @@ following patterns will work: * `IGNORE("{bar,[fz]oo}")` will ignore `bar`, `foo` and `zoo`. * `IGNORE("\\*.foo")` will ignore the literal record `*.foo`. +NOTE: `.` should not be escaped with a `\`. These are globs (like filenames), not regular expressions. + ## Typical Usage General examples: +Here we should typical usage. + {% code title="dnsconfig.js" %} ```javascript D("example.com", REG_MY_PROVIDER, DnsProvider(DSP_MY_PROVIDER), @@ -81,8 +86,11 @@ D("example.com", REG_MY_PROVIDER, DnsProvider(DSP_MY_PROVIDER), IGNORE("*", "CNAME", "dev-*"), // matches CNAMEs with targets prefixed `dev-*` IGNORE("bar", "A,MX"), // ignore only A and MX records for name bar IGNORE("*", "*", "dev-*"), // Ignore targets with a `dev-` prefix - IGNORE("*", "A", "1\.2\.3\."), // Ignore targets in the 1.2.3.0/24 CIDR block - IGNORE("*", "A", "1\.2\.3\."), // Same as previous, but records will not be reported unless `--full` + IGNORE("*", "A", "10.2.3.*"), // Ignore targets in the 10.2.3.0/24 CIDR block + IGNORE("*", "A", "10.2.3.*", true), // Same as previous line, but records will not be reported unless `--full` + IGNORE("*", "A", "10.2.*"), // Ignore targets in the 10.2.0.0/16 CIDR block + IGNORE("foo", "", "", true), // Same as first example, with `silent` set to `true`. + IGNORE("foo", "*", "*", true), // Equivalent to the previous line. END); ``` {% endcode %} @@ -290,7 +298,7 @@ D("example.com", REG_MY_PROVIDER, DnsProvider(DSP_MY_PROVIDER), {% hint style="info" %} FYI: Previously DNSControl permitted disabling this check on -a per-record basis using `IGNORE_NAME_DISABLE_SAFETY_CHECK`: +a per-record basis using `IGNORE_NAME_DISABLE_SAFETY_CHECK`. This is now a domain-level setting. {% endhint %} The `IGNORE_NAME_DISABLE_SAFETY_CHECK` feature does not exist in the diff2 @@ -314,4 +322,4 @@ as a last resort. Even then, test extensively. * There is no locking. If the external system and DNSControl make updates at the exact same time, the results are undefined. * IGNORE` works fine with records inserted into a `D()` via `D_EXTEND()`. The matching is done on the resulting FQDN of the label or target. * `targetSpec` does not match fields other than the primary target. For example, `MX` records have a target hostname plus a priority. There is no way to match the priority. -* The BIND provider can not ignore records it doesn't know about. If it does not have access to an existing zonefile, it will create a zonefile from scratch. That new zonefile will not have any external records. It will seem like they were not ignored, but in reality BIND didn't have visibility to them so that they could be ignored. +* The BIND provider can not ignore records it doesn't know about. If it does not have access to an existing zonefile, it will create a zonefile from scratch. That new zonefile will not have any external-created records. It will seem like they were not ignored, but in reality BIND will simply create a new zone without those records. diff --git a/pkg/diff2/handsoff.go b/pkg/diff2/handsoff.go index e6298b4d0c..612315d842 100644 --- a/pkg/diff2/handsoff.go +++ b/pkg/diff2/handsoff.go @@ -146,7 +146,7 @@ func genReport(recs models.Records, reason string, maxReport int) (msgs []string return nil } visibleCount, hiddenCount := countVisibility(recs) - header, footer := makeHeaderFooter(reason, !printer.SkinnyReport, maxReport, len(recs), visibleCount, hiddenCount) + header, footer := makeHeaderFooter(reason, !printer.SkinnyReport, maxReport, visibleCount, hiddenCount) msgs = append(msgs, header) msgs = append(msgs, reportMessages(recs, maxReport, !printer.SkinnyReport)...) if footer != "" { @@ -157,35 +157,41 @@ func genReport(recs models.Records, reason string, maxReport int) (msgs []string } // makeHeaderFooter generates fancy header and footer. -func makeHeaderFooter(reason string, full bool, maxReport, recsCount, visibleCount, hiddenCount int) (header, footer string) { +func makeHeaderFooter(reason string, full bool, maxReport, visibleCount, hiddenCount int) (header, footer string) { + totalCount := visibleCount + hiddenCount + + // With the --full flag, use a very simple header and no footer. if full { - // No maximum. Everything is shown. - header = fmt.Sprintf("%d records not deleted because of %s:", recsCount, reason) + header = fmt.Sprintf("FYI: %d records NOT deleted due to %s:", totalCount, reason) footer = "" + return header, footer + } - } else if visibleCount > maxReport { - // We hit the maxReport limit: - if hiddenCount > 0 { - // Some were hidden intentionally. - header = fmt.Sprintf("%d records not deleted because of %s:", recsCount, reason) - footer = fmt.Sprintf(" ...plus %d others (use --full to reveal)", recsCount-maxReport) - } else { - // Nothing hidden. - header = fmt.Sprintf("%d records not being deleted because of %s:", recsCount, reason) - footer = fmt.Sprintf(" ...%d records not displayed (use --full to show all)", recsCount-maxReport) - } + shownCount := visibleCount + if visibleCount > maxReport { + shownCount = maxReport + } - // At this point we know that the number of items being reported is less than max. - } else if visibleCount == 0 && hiddenCount != 0 { // Everything is hidden - header = fmt.Sprintf("%d records not being deleted because of %s. (Add --full to reveal)", recsCount, reason) - footer = "" - } else if hiddenCount != 0 { // Some things are hidden - header = fmt.Sprintf("%d records not being deleted because of %s:", recsCount, reason) - footer = fmt.Sprintf(" ...and %d others (use --full to reveal)", hiddenCount) - } else { // Nothing hidden - header = fmt.Sprintf("%d records not being deleted because of %s:", recsCount, reason) - footer = "" + punct := ":" + // If no records will be output, change the punctuation at the end to "." + if shownCount == 0 { + punct = "." + } + + header = fmt.Sprintf("%d records NOT deleted due to %s%s (%d/%d/%d displayed/visible/hidden)", + totalCount, reason, punct, + shownCount, visibleCount, hiddenCount, + ) + if (totalCount != shownCount) || (hiddenCount != 0) { + // Only add this if adding the flag would change the output. + header = header + " (--full shows all)" + } + + // The footer visually indicates that we've hit the maxReport limit. + footer = "" + if visibleCount != shownCount { + footer = " ..." } return header, footer @@ -217,7 +223,6 @@ func reportMessages(recs models.Records, maxReport int, full bool) (msgs []strin } for _, r := range recs { - //fmt.Printf("DEBUG: silence=%v rec=%v\n", recs[i].SilenceReporting, *recs[i]) if !r.SilenceReporting { msgs = append(msgs, genRecordMessage(r)) if len(msgs) == maxReport { diff --git a/pkg/js/parse_tests/005-ignored-records.js b/pkg/js/parse_tests/005-ignored-records.js index 25693dc95d..3f19a21915 100644 --- a/pkg/js/parse_tests/005-ignored-records.js +++ b/pkg/js/parse_tests/005-ignored-records.js @@ -18,6 +18,7 @@ D("diff2.com", "none" , IGNORE("labelc", "CNAME", "targetc") , IGNORE("silenttrue", "CNAME", "targetc", true) , IGNORE("silentfalse", "CNAME", "targetc", false) + , IGNORE("silentnull", "", "", true) // Compatibility mode: , IGNORE_NAME("nametest") , IGNORE_TARGET("targettest1") diff --git a/pkg/js/parse_tests/005-ignored-records.json b/pkg/js/parse_tests/005-ignored-records.json index 43db77f312..ba1875cc27 100644 --- a/pkg/js/parse_tests/005-ignored-records.json +++ b/pkg/js/parse_tests/005-ignored-records.json @@ -117,6 +117,10 @@ "pattern": "silentfalse", "types": "CNAME" }, + { + "pattern": "silentnull", + "types": "" + }, { "pattern": "nametest", "types": "*" @@ -175,6 +179,10 @@ "rType_pattern": "CNAME", "target_pattern": "targetc" }, + { + "label_pattern": "silentnull", + "silence_reporting": true + }, { "label_pattern": "nametest", "rType_pattern": "*" From 13e058e876f0bcdf7d7394b55d236ea993d942ed Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Tue, 26 Sep 2023 09:08:26 -0400 Subject: [PATCH 5/7] IGNORE in NO_PURGE mode should hide items from the report --- pkg/diff2/handsoff.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/diff2/handsoff.go b/pkg/diff2/handsoff.go index 612315d842..a838c5781c 100644 --- a/pkg/diff2/handsoff.go +++ b/pkg/diff2/handsoff.go @@ -248,19 +248,21 @@ func processIgnoreAndNoPurge(domain string, existing, desired, absences models.R for _, rec := range existing { isMatch, silence := matchAny(unmanagedConfigs, rec) //fmt.Printf("DEBUG: matchAny returned: %v\n", isMatch) - if isMatch { - rec.SilenceReporting = silence - ignorable = append(ignorable, rec) - } else { - if noPurge { - // Is this a candidate for purging? - if !desiredDB.ContainsLT(rec) { - // Yes, but not if it is an exception! - if !absentDB.ContainsLT(rec) { - foreign = append(foreign, rec) - } + if isMatch && silence { // Marked as silent? + rec.SilenceReporting = true + } + + if noPurge { // Process NO_PURGE. + // Is this a candidate for purging? + if !desiredDB.ContainsLT(rec) { + // Yes, but not if it is an exception! + if !absentDB.ContainsLT(rec) { + foreign = append(foreign, rec) } } + //} + } else if isMatch { // Process IGNORE() matches. + ignorable = append(ignorable, rec) } } return ignorable, foreign From 88a4ac59d81ff9ee439efe67ec3755597f1ac405 Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Tue, 26 Sep 2023 09:42:54 -0400 Subject: [PATCH 6/7] fix tests --- pkg/diff2/handsoff_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/diff2/handsoff_test.go b/pkg/diff2/handsoff_test.go index ce2161cd1c..20d0c67a37 100644 --- a/pkg/diff2/handsoff_test.go +++ b/pkg/diff2/handsoff_test.go @@ -184,7 +184,7 @@ D("f.com", "none", IGNORE_NAME("foo3"), {}) ` - handsoffHelper(t, existingZone, desiredJs, true, ` + handsoffHelper(t, existingZone, desiredJs, false, ` IGNORED: foo3 A 3.3.3.3 foo3 MX 10 mymx.example.com. @@ -207,7 +207,7 @@ D("f.com", "none", IGNORE_NAME("foo3", "MX"), {}) ` - handsoffHelper(t, existingZone, desiredJs, true, ` + handsoffHelper(t, existingZone, desiredJs, false, ` IGNORED: foo3 MX 10 mymx.example.com. FOREIGN: @@ -228,7 +228,7 @@ D("f.com", "none", IGNORE_TARGET('**.acm-validations.aws.', 'CNAME'), {}) ` - handsoffHelper(t, existingZone, desiredJs, true, ` + handsoffHelper(t, existingZone, desiredJs, false, ` IGNORED: _2222222222222222.cr CNAME _333333.nnn.acm-validations.aws. FOREIGN: From 958fd0a20472a275f5ae50c70026000fa8f22bad Mon Sep 17 00:00:00 2001 From: Tom Limoncelli Date: Mon, 26 Feb 2024 12:26:23 -0500 Subject: [PATCH 7/7] Update documentation/functions/domain/IGNORE.md Co-authored-by: Jeffrey Cafferata --- documentation/functions/domain/IGNORE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/functions/domain/IGNORE.md b/documentation/functions/domain/IGNORE.md index 101b511886..f85fa949f9 100644 --- a/documentation/functions/domain/IGNORE.md +++ b/documentation/functions/domain/IGNORE.md @@ -69,7 +69,9 @@ following patterns will work: * `IGNORE("{bar,[fz]oo}")` will ignore `bar`, `foo` and `zoo`. * `IGNORE("\\*.foo")` will ignore the literal record `*.foo`. -NOTE: `.` should not be escaped with a `\`. These are globs (like filenames), not regular expressions. +{% hint style="info" %} +**NOTE**: `.` should not be escaped with a `\`. These are globs (like filenames), not regular expressions. +{% endhint %} ## Typical Usage