Auto 4599/extended protocol testing (#282)

* add protocol modifier functions for testing purposes

* added README

* isolate make functions to pkg, internal, and cmd

* use reduced set of files for make generate

Makefile

@@ -1,13 +1,13 @@
 GOBASE=$(shell pwd)
 GOBIN=$(GOBASE)/bin
 
-GOPACKAGES = $(shell go list ./...)
+GOPACKAGES = $(shell go list ./pkg/... && go list ./internal/... && go list ./cmd/...)
 
 dependencies:
 	go mod download
 
 generate:
-	go generate -x ./...
+	go generate -x $(GOPACKAGES)
 
 test: dependencies
 	@go test -v $(GOPACKAGES)

tools/testprotocol/README.md

@@ -0,0 +1,87 @@
+# Protocol Testing
+
+The contents of this package are pre-built tools that aid in testing the OCR Automation protocol where one or more nodes
+have modified outputs that are in conflict with non-modified nodes. This allows for asserting that the protocol performs
+as expected when integrated in a network environment of un-trusted nodes.
+
+## Direct Modifiers
+
+Direct modifiers apply data changes directly before or directly after encoding of either observations, outcomes, or
+reports. This output modification strategy allows for encoding changes outside the scope of strict Golang types as well
+as simple value modifications directly on the output type.
+
+The subpackage `modify` contains the general modifier structure as well as some pre-built modifiers and collection
+constants. There are two modifier variants:
+
+- `Modify`: takes a modifier input such as `AsObservation` or `AsOutcome` which apply type assertions on the subsequent modifiers
+- `ModifyBytes`: takes a slice of `MapModifier` where key/value pairs are provided to modifiers
+
+### Modify
+
+Use the `Modify` type when applying modifications directly to a type such as `AutomationOutcome`. Multiple usage
+examples are located in `modify/defaults.go`. To write new modifiers, follow the pattern below:
+
+```
+// WithPerformableBlockAs adds the provided block number within the scope of a PerformableModifier function
+func WithPerformableBlockAs(block types.BlockNumber) PerformableModifier {
+	return func(performables []types.CheckResult) []types.CheckResult {
+        // the block number in scope is applied to all performables
+		for _, performable := range performables {
+			performable.Trigger.BlockNumber = block
+		}
+
+        // the modified performables are returned back to the observation or outcome
+		return performables
+	}
+}
+```
+
+Use this modifier in multiple typed modifiers as a composible function:
+
+```
+// use the above function to modify performables in observations
+Modify(
+    "set all performables to block 1",
+    AsObservation(
+        WithPerformableBlockAs(types.BlockNumber(1))))
+
+// use the above function to modify performables in outcomes
+Modify(
+    "set all performables to block 1",
+    AsOutcome(
+        WithPerformableBlockAs(types.BlockNumber(1))))
+```
+
+### ModifyBytes
+
+This modify function can be used to change values directly in a json encoded output. Instead of operating on direct
+types like `AutomationOutcome`, the json input is split into key/value pairs before being passed to subsequent custom
+modifiers. Write a new modifier using the following pattern:
+
+```
+// WithModifyKeyValue is a generic key/value modifier where a key and modifier function are provided and the function
+// recursively searches the json path for the provided key and modifies the value when the key is found.
+func WithModifyKeyValue(key string, fn ValueModifierFunc) MapModifier {
+	return func(ctx context.Context, values map[string]interface{}, err error) (map[string]interface{}, error) {
+		return recursiveModify(key, "root", fn, values), err
+	}
+}
+```
+
+Use this modifier as a generic key/value modifer for arbitrary json structures:
+
+```
+ModifyBytes(
+    "set block value to very large number as string",
+    WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+        return "98989898989898989898989898989898989898989898"
+    }))
+```
+
+## Indirect Modifiers
+
+In many cases, data modifications must be applied BEFORE an observation or outcome can be constructed. These types of
+cases might include repeated proposals where state between rounds might need to be tracked and specific data needs to be
+captured and re-broadcast where the unmodified protocol wouldn't. 
+
+Specifics TBD

tools/testprotocol/modify/byte.go

@@ -0,0 +1,65 @@
+package modify
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+)
+
+type NamedByteModifier func(context.Context, []byte, error) (string, []byte, error)
+type MapModifier func(context.Context, map[string]interface{}, error) (map[string]interface{}, error)
+type ValueModifierFunc func(string, interface{}) interface{}
+
+// WithModifyKeyValue recursively operates on all key-value pairs in the provided map and applies the provided modifier
+// function if the key matches. The path provided to the modifier function starts with `root` and is appended with every
+// key encountered in the tree. ex: `root.someKey.anotherKey`.
+func WithModifyKeyValue(key string, fn ValueModifierFunc) MapModifier {
+	return func(ctx context.Context, values map[string]interface{}, err error) (map[string]interface{}, error) {
+		return recursiveModify(key, "root", fn, values), err
+	}
+}
+
+// ModifyBytes deconstructs provided bytes into a map[string]interface{} and passes the decoded map to provided
+// modifiers. The final modified map is re-encoded as bytes and returned by the modifier function.
+func ModifyBytes(name string, modifiers ...MapModifier) NamedByteModifier {
+	return func(ctx context.Context, bytes []byte, err error) (string, []byte, error) {
+		var values map[string]interface{}
+
+		if err := json.Unmarshal(bytes, &values); err != nil {
+			return name, bytes, err
+		}
+
+		for _, modifier := range modifiers {
+			values, err = modifier(ctx, values, err)
+		}
+
+		bytes, err = json.Marshal(values)
+
+		return name, bytes, err
+	}
+}
+
+func recursiveModify(key, path string, mod ValueModifierFunc, values map[string]interface{}) map[string]interface{} {
+	for mapKey, mapValue := range values {
+		newPath := fmt.Sprintf("%s.%s", path, mapKey)
+
+		switch nextValues := mapValue.(type) {
+		case map[string]interface{}:
+			values[key] = recursiveModify(key, newPath, mod, nextValues)
+		case []interface{}:
+			for idx, arrayValue := range nextValues {
+				newPath = fmt.Sprintf("%s[%d]", newPath, idx)
+
+				if mappedArray, ok := arrayValue.(map[string]interface{}); ok {
+					nextValues[idx] = recursiveModify(key, newPath, mod, mappedArray)
+				}
+			}
+		default:
+			if mapKey == key {
+				values[key] = mod(newPath, mapValue)
+			}
+		}
+	}
+
+	return values
+}

tools/testprotocol/modify/byte_test.go

@@ -0,0 +1,50 @@
+package modify_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3"
+	ocr2keeperstypes "github.com/smartcontractkit/ocr2keepers/pkg/v3/types"
+	"github.com/smartcontractkit/ocr2keepers/tools/testprotocol/modify"
+)
+
+func TestModifyBytes(t *testing.T) {
+	originalName := "test modifier"
+	modifier := modify.ModifyBytes(
+		originalName,
+		modify.WithModifyKeyValue(
+			"BlockNumber",
+			func(path string, values interface{}) interface{} {
+				return -1
+			}))
+
+	observation := ocr2keepers.AutomationObservation{
+		Performable: []ocr2keeperstypes.CheckResult{
+			{
+				Trigger: ocr2keeperstypes.NewLogTrigger(
+					ocr2keeperstypes.BlockNumber(10),
+					[32]byte{},
+					&ocr2keeperstypes.LogTriggerExtension{
+						TxHash:      [32]byte{},
+						Index:       1,
+						BlockHash:   [32]byte{},
+						BlockNumber: ocr2keeperstypes.BlockNumber(10),
+					},
+				),
+			},
+		},
+		UpkeepProposals: []ocr2keeperstypes.CoordinatedBlockProposal{},
+		BlockHistory:    []ocr2keeperstypes.BlockKey{},
+	}
+
+	original, err := json.Marshal(observation)
+	name, modified, err := modifier(context.Background(), original, err)
+
+	assert.NoError(t, err)
+	assert.NotEqual(t, original, modified)
+	assert.Equal(t, originalName, name)
+}

tools/testprotocol/modify/defaults.go

@@ -0,0 +1,107 @@
+package modify
+
+import "github.com/smartcontractkit/ocr2keepers/pkg/v3/types"
+
+var (
+	ObservationModifiers = []NamedModifier{
+		Modify(
+			"set all proposals to block 1",
+			AsObservation(
+				WithProposalBlockAs(types.BlockNumber(1)))),
+		Modify(
+			"set all proposals to block 1_000_000_000",
+			AsObservation(
+				WithProposalBlockAs(types.BlockNumber(1_000_000_000)))),
+		Modify(
+			"set all proposals to block 0",
+			AsObservation(
+				WithProposalBlockAs(types.BlockNumber(0)))),
+		Modify(
+			"set all performables to block 1",
+			AsObservation(
+				WithPerformableBlockAs(types.BlockNumber(1)))),
+		Modify(
+			"set all performables to block 1_000_000_000",
+			AsObservation(
+				WithPerformableBlockAs(types.BlockNumber(1_000_000_000)))),
+		Modify(
+			"set all performables to block 0",
+			AsObservation(
+				WithPerformableBlockAs(types.BlockNumber(0)))),
+		Modify(
+			"set all block history numbers to 0",
+			AsObservation(
+				WithBlockHistoryBlockAs(0))),
+		Modify(
+			"set all block history numbers to 1",
+			AsObservation(
+				WithBlockHistoryBlockAs(1))),
+		Modify(
+			"set all block history numbers to 1_000_000_000",
+			AsObservation(
+				WithBlockHistoryBlockAs(1_000_000_000))),
+	}
+
+	OutcomeModifiers = []NamedModifier{
+		Modify(
+			"set all proposals to block 1",
+			AsOutcome(
+				WithProposalBlockAs(types.BlockNumber(1)))),
+		Modify(
+			"set all proposals to block 1_000_000_000",
+			AsOutcome(
+				WithProposalBlockAs(types.BlockNumber(1_000_000_000)))),
+		Modify(
+			"set all proposals to block 0",
+			AsOutcome(
+				WithProposalBlockAs(types.BlockNumber(0)))),
+		Modify(
+			"set all performables to block 1",
+			AsOutcome(
+				WithPerformableBlockAs(types.BlockNumber(1)))),
+		Modify(
+			"set all performables to block 1_000_000_000",
+			AsOutcome(
+				WithPerformableBlockAs(types.BlockNumber(1_000_000_000)))),
+		Modify(
+			"set all performables to block 0",
+			AsOutcome(
+				WithPerformableBlockAs(types.BlockNumber(0)))),
+	}
+
+	ObservationInvalidValueModifiers = []NamedByteModifier{
+		ModifyBytes(
+			"set block value to empty string",
+			WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+				return ""
+			})),
+		ModifyBytes(
+			"set block value to negative number",
+			WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+				return -1
+			})),
+		ModifyBytes(
+			"set block value to very large number as string",
+			WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+				return "98989898989898989898989898989898989898989898"
+			})),
+	}
+
+	InvalidBlockModifiers = []NamedByteModifier{
+		ModifyBytes(
+			"set block value to empty string",
+			WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+				return ""
+			})),
+		ModifyBytes(
+			"set block value to negative number",
+			WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+				return -1
+			})),
+		ModifyBytes(
+			"set block value to very large number as string",
+			WithModifyKeyValue("BlockNumber", func(_ string, value interface{}) interface{} {
+				return "98989898989898989898989898989898989898989898"
+			})),
+	}
+)