Automation LOOPP services (#11631)

* Import from chainlink-common instead of chainlink-automation
Update chainlink-automation
Update the relayer to expose services
Reduce the services created outside LOOPP

Bump dependencies

This reverts commit aaed53e0be87c94918471d57b8144f0fca7ed5dd.

Pass provider services into the delegate conf

Use more provider services, bump common

Move MercuryCredentials into common

Clean up the old automation services for 2.1

Bump common and automation

Make modgraph

Pass mercury credentials directly

* Bump automation and common

* Bump testing framework

* Bump common and automation to latest main

core/config/mercury_config.go

@@ -3,7 +3,7 @@ package config
 import (
 	"time"
 
-	ocr2models "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models"
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
 )
 
 type MercuryCache interface {
@@ -17,7 +17,7 @@ type MercuryTLS interface {
 }
 
 type Mercury interface {
-	Credentials(credName string) *ocr2models.MercuryCredentials
+	Credentials(credName string) *types.MercuryCredentials
 	Cache() MercuryCache
 	TLS() MercuryTLS
 }

core/scripts/chaincli/handler/debug.go

@@ -14,13 +14,15 @@ import (
 	"os"
 	"strconv"
 
+	types2 "github.com/smartcontractkit/chainlink-common/pkg/types"
+
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	gethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 
-	ocr2keepers "github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	evm21 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21"
 
@@ -31,7 +33,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1"
 	iregistry21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
-	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/encoding"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/mercury"
@@ -256,7 +257,7 @@ func (k *Keeper) Debug(ctx context.Context, args []string) {
 	}
 
 	if checkResult.UpkeepFailureReason == uint8(encoding.UpkeepFailureReasonTargetCheckReverted) {
-		mc := &models.MercuryCredentials{LegacyURL: k.cfg.MercuryLegacyURL, URL: k.cfg.MercuryURL, Username: k.cfg.MercuryID, Password: k.cfg.MercuryKey}
+		mc := &types2.MercuryCredentials{LegacyURL: k.cfg.MercuryLegacyURL, URL: k.cfg.MercuryURL, Username: k.cfg.MercuryID, Password: k.cfg.MercuryKey}
 		mercuryConfig := evm21.NewMercuryConfig(mc, core.StreamsCompatibleABI)
 		lggr, _ := logger.NewLogger()
 		blockSub := &blockSubscriber{k.client}

core/scripts/go.mod

@@ -18,8 +18,8 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pelletier/go-toml/v2 v2.1.1
 	github.com/shopspring/decimal v1.3.1
-	github.com/smartcontractkit/chainlink-automation v1.0.1
-	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240108182329-237f56daad6b
+	github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240118014648-1ab6a88c9429
+	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240118012339-4864e2306bb1
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20231130143053-c5102a9c0fb7

core/scripts/go.sum

@@ -1162,10 +1162,10 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 h1:T3lFWumvbfM1u/etVq42Afwq/jtNSBSOA8n5jntnNPo=
 github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704/go.mod h1:2QuJdEouTWjh5BDy5o/vgGXQtR4Gz8yH1IYB5eT7u4M=
-github.com/smartcontractkit/chainlink-automation v1.0.1 h1:vVjBFq2Zsz21kPy1Pb0wpjF9zrbJX+zjXphDeeR4XZk=
-github.com/smartcontractkit/chainlink-automation v1.0.1/go.mod h1:INSchkV3ntyDdlZKGWA030MPDpp6pbeuiRkRKYFCm2k=
-github.com/smartcontractkit/chainlink-common v0.1.7-0.20240108182329-237f56daad6b h1:PRuvHgPka1gIGPKASuJHvqUvYWhZO7z5B/kKapjkZaM=
-github.com/smartcontractkit/chainlink-common v0.1.7-0.20240108182329-237f56daad6b/go.mod h1:f+0ei9N4PlTJHu7pbGzEjTnBUr45syPdGFu5+31lS5Q=
+github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240118014648-1ab6a88c9429 h1:xkejUBZhcBpBrTSfxc91Iwzadrb6SXw8ks69bHIQ9Ww=
+github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240118014648-1ab6a88c9429/go.mod h1:wJmVvDf4XSjsahWtfUq3wvIAYEAuhr7oxmxYnEL/LGQ=
+github.com/smartcontractkit/chainlink-common v0.1.7-0.20240118012339-4864e2306bb1 h1:3cWO2/lFVDul5SVTgl4/RX/GXcT8Zq5NGMPeNEz09tY=
+github.com/smartcontractkit/chainlink-common v0.1.7-0.20240118012339-4864e2306bb1/go.mod h1:f+0ei9N4PlTJHu7pbGzEjTnBUr45syPdGFu5+31lS5Q=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231218175426-6e0427c661e5 h1:kBnmjv3fxU7krVIqZFvo1m4F6qBc4vPURQFX/mcChhI=
 github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231218175426-6e0427c661e5/go.mod h1:EoM7wQ81mov7wsUzG4zEnnr0EH0POEo/I0hRDg433TU=
 github.com/smartcontractkit/chainlink-data-streams v0.0.0-20231204152908-a6e3fe8ff2a1 h1:xYqRgZO0nMSO8CBCMR0r3WA+LZ4kNL8a6bnbyk/oBtQ=

core/services/chainlink/config_mercury.go

@@ -3,9 +3,10 @@ package chainlink
 import (
 	"time"
 
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
+
 	"github.com/smartcontractkit/chainlink/v2/core/config"
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
-	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models"
 )
 
 var _ config.MercuryCache = (*mercuryCacheConfig)(nil)
@@ -37,9 +38,9 @@ type mercuryConfig struct {
 	s toml.MercurySecrets
 }
 
-func (m *mercuryConfig) Credentials(credName string) *models.MercuryCredentials {
+func (m *mercuryConfig) Credentials(credName string) *types.MercuryCredentials {
 	if mc, ok := m.s.Credentials[credName]; ok {
-		c := &models.MercuryCredentials{
+		c := &types.MercuryCredentials{
 			URL:      mc.URL.URL().String(),
 			Password: string(*mc.Password),
 			Username: string(*mc.Username),

core/services/chainlink/config_mercury_test.go

@@ -3,11 +3,11 @@ package chainlink
 import (
 	"testing"
 
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models"
-
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
 )
 
@@ -33,8 +33,8 @@ func TestMercuryConfig(t *testing.T) {
 	require.NoError(t, err)
 
 	m := cfg.Mercury()
-	assert.Equal(t, &models.MercuryCredentials{URL: "https://chain1.link", Username: "username1", Password: "password1"}, m.Credentials("cred1"))
-	assert.Equal(t, &models.MercuryCredentials{URL: "https://chain2.link", Username: "username2", Password: "password2"}, m.Credentials("cred2"))
+	assert.Equal(t, &types.MercuryCredentials{URL: "https://chain1.link", Username: "username1", Password: "password1"}, m.Credentials("cred1"))
+	assert.Equal(t, &types.MercuryCredentials{URL: "https://chain2.link", Username: "username2", Password: "password2"}, m.Credentials("cred2"))
 }
 
 func TestMercuryTLS(t *testing.T) {

core/services/ocr2/delegate.go

@@ -43,7 +43,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/job"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ocr2key"
-	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/dkg"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/dkg/persistence"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/functions"
@@ -188,7 +187,7 @@ type jobPipelineConfig interface {
 }
 
 type mercuryConfig interface {
-	Credentials(credName string) *models.MercuryCredentials
+	Credentials(credName string) *types.MercuryCredentials
 	Cache() coreconfig.MercuryCache
 	TLS() coreconfig.MercuryTLS
 }
@@ -1089,12 +1088,13 @@ func (d *Delegate) newServicesOCR2Keepers21(
 
 	provider, err := relayer.NewPluginProvider(ctx,
 		types.RelayArgs{
-			ExternalJobID: jb.ExternalJobID,
-			JobID:         jb.ID,
-			ContractID:    spec.ContractID,
-			New:           d.isNewlyCreatedJob,
-			RelayConfig:   spec.RelayConfig.Bytes(),
-			ProviderType:  string(spec.PluginType),
+			ExternalJobID:      jb.ExternalJobID,
+			JobID:              jb.ID,
+			ContractID:         spec.ContractID,
+			New:                d.isNewlyCreatedJob,
+			RelayConfig:        spec.RelayConfig.Bytes(),
+			ProviderType:       string(spec.PluginType),
+			MercuryCredentials: mc,
 		}, types.PluginArgs{
 			TransmitterID: transmitterID,
 			PluginConfig:  spec.PluginConfig.Bytes(),
@@ -1108,12 +1108,7 @@ func (d *Delegate) newServicesOCR2Keepers21(
 		return nil, errors.New("could not coerce PluginProvider to AutomationProvider")
 	}
 
-	chain, err := d.legacyChains.Get(rid.ChainID)
-	if err != nil {
-		return nil, fmt.Errorf("keeper2 services: failed to get chain %s: %w", rid.ChainID, err)
-	}
-
-	services, err := ocr2keeper.EVMDependencies21(jb, d.db, lggr, chain, mc, kb, d.cfg.Database())
+	services, err := ocr2keeper.EVMDependencies21(kb)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not build dependencies for ocr2 keepers")
 	}
@@ -1154,15 +1149,15 @@ func (d *Delegate) newServicesOCR2Keepers21(
 		OffchainKeyring:              kb,
 		OnchainKeyring:               services.Keyring(),
 		LocalConfig:                  lc,
-		LogProvider:                  services.LogEventProvider(),
-		EventProvider:                services.TransmitEventProvider(),
-		Runnable:                     services.Registry(),
-		Encoder:                      services.Encoder(),
-		BlockSubscriber:              services.BlockSubscriber(),
-		RecoverableProvider:          services.LogRecoverer(),
-		PayloadBuilder:               services.PayloadBuilder(),
-		UpkeepProvider:               services.UpkeepProvider(),
-		UpkeepStateUpdater:           services.UpkeepStateStore(),
+		LogProvider:                  keeperProvider.LogEventProvider(),
+		EventProvider:                keeperProvider.TransmitEventProvider(),
+		Runnable:                     keeperProvider.Registry(),
+		Encoder:                      keeperProvider.Encoder(),
+		BlockSubscriber:              keeperProvider.BlockSubscriber(),
+		RecoverableProvider:          keeperProvider.LogRecoverer(),
+		PayloadBuilder:               keeperProvider.PayloadBuilder(),
+		UpkeepProvider:               keeperProvider.UpkeepProvider(),
+		UpkeepStateUpdater:           keeperProvider.UpkeepStateStore(),
 		UpkeepTypeGetter:             ocr2keeper21core.GetUpkeepType,
 		WorkIDGenerator:              ocr2keeper21core.UpkeepWorkID,
 		// TODO: Clean up the config
@@ -1179,12 +1174,12 @@ func (d *Delegate) newServicesOCR2Keepers21(
 
 	automationServices := []job.ServiceCtx{
 		keeperProvider,
-		services.Registry(),
-		services.BlockSubscriber(),
-		services.LogEventProvider(),
-		services.LogRecoverer(),
-		services.UpkeepStateStore(),
-		services.TransmitEventProvider(),
+		keeperProvider.Registry(),
+		keeperProvider.BlockSubscriber(),
+		keeperProvider.LogEventProvider(),
+		keeperProvider.LogRecoverer(),
+		keeperProvider.UpkeepStateStore(),
+		keeperProvider.TransmitEventProvider(),
 		pluginService,
 	}
 
@@ -1194,7 +1189,7 @@ func (d *Delegate) newServicesOCR2Keepers21(
 		customTelemService, custErr := autotelemetry21.NewAutomationCustomTelemetryService(
 			endpoint,
 			lggr,
-			services.BlockSubscriber(),
+			keeperProvider.BlockSubscriber(),
 			keeperProvider.ContractConfigTracker(),
 		)
 		if custErr != nil {
@@ -1230,7 +1225,7 @@ func (d *Delegate) newServicesOCR2Keepers20(
 		return nil, fmt.Errorf("keepers2.0 services: failed to get chain (%s): %w", rid.ChainID, err2)
 	}
 
-	keeperProvider, rgstry, encoder, logProvider, err2 := ocr2keeper.EVMDependencies20(jb, d.db, lggr, chain, d.ethKs)
+	keeperProvider, rgstry, encoder, logProvider, err2 := ocr2keeper.EVMDependencies20(jb, d.db, lggr, chain, d.ethKs, d.cfg.Database())
 	if err2 != nil {
 		return nil, errors.Wrap(err2, "could not build dependencies for ocr2 keepers")
 	}

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go

@@ -4,7 +4,9 @@ import (
 	"math/big"
 	"sync"
 
-	ocr2keepers "github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+	"github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+
+	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
 )
@@ -18,7 +20,7 @@ type ActiveUpkeepList interface {
 	// Remove removes entries from the list
 	Remove(id ...*big.Int) int
 	// View returns the list of IDs of the given type
-	View(...ocr2keepers.UpkeepType) []*big.Int
+	View(...types.UpkeepType) []*big.Int
 	// IsActive returns true if the given ID is of an active upkeep
 	IsActive(id *big.Int) bool
 	Size() int
@@ -81,7 +83,7 @@ func (al *activeList) Remove(ids ...*big.Int) int {
 }
 
 // View returns the list of IDs of the given type
-func (al *activeList) View(upkeepTypes ...ocr2keepers.UpkeepType) []*big.Int {
+func (al *activeList) View(upkeepTypes ...types.UpkeepType) []*big.Int {
 	al.lock.RLock()
 	defer al.lock.RUnlock()
 

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list_test.go

@@ -5,27 +5,29 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+
 	"github.com/stretchr/testify/require"
 
-	ocr2keepers "github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
 )
 
 func TestActiveUpkeepList(t *testing.T) {
 	logIDs := []ocr2keepers.UpkeepIdentifier{
-		core.GenUpkeepID(ocr2keepers.LogTrigger, "0"),
-		core.GenUpkeepID(ocr2keepers.LogTrigger, "1"),
-		core.GenUpkeepID(ocr2keepers.LogTrigger, "2"),
-		core.GenUpkeepID(ocr2keepers.LogTrigger, "3"),
-		core.GenUpkeepID(ocr2keepers.LogTrigger, "4"),
+		core.GenUpkeepID(types.LogTrigger, "0"),
+		core.GenUpkeepID(types.LogTrigger, "1"),
+		core.GenUpkeepID(types.LogTrigger, "2"),
+		core.GenUpkeepID(types.LogTrigger, "3"),
+		core.GenUpkeepID(types.LogTrigger, "4"),
 	}
 	conditionalIDs := []ocr2keepers.UpkeepIdentifier{
-		core.GenUpkeepID(ocr2keepers.ConditionTrigger, "0"),
-		core.GenUpkeepID(ocr2keepers.ConditionTrigger, "1"),
-		core.GenUpkeepID(ocr2keepers.ConditionTrigger, "2"),
-		core.GenUpkeepID(ocr2keepers.ConditionTrigger, "3"),
-		core.GenUpkeepID(ocr2keepers.ConditionTrigger, "4"),
+		core.GenUpkeepID(types.ConditionTrigger, "0"),
+		core.GenUpkeepID(types.ConditionTrigger, "1"),
+		core.GenUpkeepID(types.ConditionTrigger, "2"),
+		core.GenUpkeepID(types.ConditionTrigger, "3"),
+		core.GenUpkeepID(types.ConditionTrigger, "4"),
 	}
 
 	tests := []struct {
@@ -70,15 +72,15 @@ func TestActiveUpkeepList(t *testing.T) {
 			for _, id := range tc.remove {
 				require.False(t, al.IsActive(id))
 			}
-			logIds := al.View(ocr2keepers.LogTrigger)
+			logIds := al.View(types.LogTrigger)
 			require.Equal(t, len(tc.expectedLogIds), len(logIds))
 			sort.Slice(logIds, func(i, j int) bool {
 				return logIds[i].Cmp(logIds[j]) < 0
 			})
 			for i := range logIds {
 				require.Equal(t, tc.expectedLogIds[i], logIds[i])
 			}
-			conditionalIds := al.View(ocr2keepers.ConditionTrigger)
+			conditionalIds := al.View(types.ConditionTrigger)
 			require.Equal(t, len(tc.expectedConditionalIds), len(conditionalIds))
 			sort.Slice(conditionalIds, func(i, j int) bool {
 				return conditionalIds[i].Cmp(conditionalIds[j]) < 0
@@ -98,7 +100,7 @@ func TestActiveUpkeepList_error(t *testing.T) {
 		al.items["-1"] = true
 		al.items["100"] = true
 
-		keys := al.View(ocr2keepers.ConditionTrigger)
+		keys := al.View(types.ConditionTrigger)
 		require.Equal(t, []*big.Int{big.NewInt(100)}, keys)
 	})
 }

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/autotelemetry21/custom_telemetry.go

@@ -10,12 +10,11 @@ import (
 	"github.com/smartcontractkit/libocr/commontypes"
 	"github.com/smartcontractkit/libocr/offchainreporting2plus/types"
 
-	ocr2keepers "github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
 
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
-	evm21 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21"
 	"github.com/smartcontractkit/chainlink/v2/core/services/synchronization/telem"
 	"github.com/smartcontractkit/chainlink/v2/core/static"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -24,7 +23,7 @@ import (
 type AutomationCustomTelemetryService struct {
 	services.StateMachine
 	monitoringEndpoint    commontypes.MonitoringEndpoint
-	blockSubscriber       *evm21.BlockSubscriber
+	blockSubscriber       ocr2keepers.BlockSubscriber
 	blockSubChanID        int
 	threadCtrl            utils.ThreadControl
 	lggr                  logger.Logger
@@ -34,7 +33,7 @@ type AutomationCustomTelemetryService struct {
 
 // NewAutomationCustomTelemetryService creates a telemetry service for new blocks and node version
 func NewAutomationCustomTelemetryService(me commontypes.MonitoringEndpoint,
-	lggr logger.Logger, blocksub *evm21.BlockSubscriber, configTracker types.ContractConfigTracker) (*AutomationCustomTelemetryService, error) {
+	lggr logger.Logger, blocksub ocr2keepers.BlockSubscriber, configTracker types.ContractConfigTracker) (*AutomationCustomTelemetryService, error) {
 	return &AutomationCustomTelemetryService{
 		monitoringEndpoint:    me,
 		threadCtrl:            utils.NewThreadControl(),

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/block_subscriber.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 
-	ocr2keepers "github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
 

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/block_subscriber_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
-	ocr2keepers "github.com/smartcontractkit/chainlink-automation/pkg/v3/types"
+	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	commonmocks "github.com/smartcontractkit/chainlink/v2/common/mocks"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types"