Add (#1275)

# Related Github tickets

- VolumeFi#2061

# Background

- Fixes the Skyway nonce reset implementation during a compass ugprade
- Introduces a governance proposal message which may be used to override
the last nonce across all validators for a given chain to recover from a
desync

# Testing completed

- [x] test coverage exists or has been added/updated
- [x] tested in a private testnet

# Breaking changes

- [x] I have checked my code for breaking changes
- [x] If there are breaking changes, there is a supporting migration.

proto/palomachain/paloma/skyway/msgs.proto

@@ -65,6 +65,18 @@ service Msg {
  option (google.api.http).post =
  "/palomachain/paloma/skyway/light-node-sale-claim";
  }
+
+ rpc OverrideNonceProposal(MsgNonceOverrideProposal) returns (google.protobuf.Empty);
+}
+
+// TODO: Remove this message after usage
+message MsgNonceOverrideProposal {
+ option deprecated = true;
+ option (cosmos.msg.v1.signer) = "metadata";
+ palomachain.paloma.valset.MsgMetadata metadata = 1
+ [ (gogoproto.nullable) = false ];
+ string chain_reference_id = 2;
+ uint64 nonce = 3;
 }
 
 // MsgSendToRemote

x/skyway/client/cli/query.go

@@ -35,6 +35,7 @@ func GetQueryCmd() *cobra.Command {
  CmdGetAttestations(),
  CmdGetLastObservedEthBlock(),
  CmdGetLastObservedEthNonce(),
+ CmdGetLastObservedEthNoncesByValAddress(),
  CmdGetQueryParams(),
  CmdGetQueryBridgeTax(),
  CmdGetQueryBridgeTransferLimits(),
@@ -307,6 +308,39 @@ func CmdGetLastObservedEthNonce() *cobra.Command {
  return cmd
 }
 
+func CmdGetLastObservedEthNoncesByValAddress() *cobra.Command {
+ short := "Query the last observed event nonce on the remote chain."
+ long := short + "\n\nThis value is expected to lag behind actual remote block height significantly due to 1. Remote Chain Finality and 2. Consensus mirroring the state."
+
+ // nolint: exhaustruct
+ cmd := &cobra.Command{
+ Use: "last-observed-nonce-by-val [chain-reference-id] [val-addr]",
+ Short: short,
+ Long: long,
+ Args: cobra.ExactArgs(2),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ clientCtx, err := client.GetClientQueryContext(cmd)
+ if err != nil {
+ return err
+ }
+ queryClient := types.NewQueryClient(clientCtx)
+
+ req := &types.QueryLastObservedSkywayNonceByAddrRequest{
+ Address: args[1],
+ ChainReferenceId: args[0],
+ }
+ res, err := queryClient.LastObservedSkywayNonceByAddr(cmd.Context(), req)
+ if err != nil {
+ return err
+ }
+
+ return clientCtx.PrintProto(res)
+ },
+ }
+ flags.AddQueryFlagsToCmd(cmd)
+ return cmd
+}
+
 // CmdGetQueryParams fetches the current Skyway module params
 func CmdGetQueryParams() *cobra.Command {
  // nolint: exhaustruct

x/skyway/keeper/keeper.go

@@ -93,33 +93,8 @@ func NewKeeper(
  eventbus.EVMActivatedChain().Subscribe(
  "skyway-keeper",
  func(ctx context.Context, e eventbus.EVMActivatedChainEvent) error {
- logger := liblog.FromKeeper(ctx, k).
- WithComponent("skyway-activated-chain-callback").
- WithFields(
- "chain_reference_id", e.ChainReferenceID,
- "smart_contract_unique_id", string(e.SmartContractUniqueID),
- )
-
  k.setLatestCompassID(ctx, e.ChainReferenceID, string(e.SmartContractUniqueID))
-
- err := k.setLastObservedSkywayNonce(ctx, e.ChainReferenceID, 0)
- if err != nil {
- logger.WithError(err).Warn("Failed to reset skyway nonce")
- return err
- }
-
- err = k.IterateValidatorLastEventNonces(ctx, e.ChainReferenceID, func(key []byte, _ uint64) bool {
- store := k.GetStore(ctx, e.ChainReferenceID)
- store.Delete(key)
- return false
- })
- if err != nil {
- logger.WithError(err).Warn("Failed to reset validator skyway nonces")
- return err
- }
-
- logger.Info("Updated last observed nonce successfully")
- return nil
+ return k.overrideNonce(ctx, e.ChainReferenceID, 0)
  })
 
  return k
@@ -474,3 +449,34 @@ func (k Keeper) SetAllLighNodeSaleContracts(
 
  return nil
 }
+
+// overrideNonce purposefully circumvents keeper setters and getters, as those perform
+// some biased operations and may end up inserting a different value or dropping it altogether.
+// it should never be used outside of the EVM activation event handler or a
+// proposal.
+func (k Keeper) overrideNonce(ctx context.Context, chainReferenceId string, nonce uint64) error {
+ cacheCtx, commit := sdk.UnwrapSDKContext(ctx).CacheContext()
+ logger := liblog.FromKeeper(ctx, k).WithComponent("nonce-override")
+
+ store := k.GetStore(ctx, chainReferenceId)
+ store.Set(types.LastObservedEventNonceKey, types.UInt64Bytes(nonce))
+
+ keys := [][]byte{}
+ err := k.IterateValidatorLastEventNonces(cacheCtx, chainReferenceId, func(key []byte, _ uint64) bool {
+ keys = append(keys, key)
+ return false
+ })
+ if err != nil {
+ logger.WithError(err).Warn("Failed to reset validator skyway nonces")
+ return err
+ }
+
+ for _, key := range keys {
+ prefixStore := prefix.NewStore(store, types.LastEventNonceByValidatorKey)
+ prefixStore.Set(key, types.UInt64Bytes(nonce))
+ }
+
+ commit()
+ logger.Info("Updated last observed nonce successfully")
+ return nil
+}

x/skyway/keeper/msg_server.go

@@ -416,3 +416,10 @@ func (k msgServer) EstimateBatchGas(
  },
  )
 }
+
+func (k *msgServer) OverrideNonceProposal(ctx context.Context, req *types.MsgNonceOverrideProposal) (*emptypb.Empty, error) {
+ if req.Metadata.Creator != k.authority {
+ return nil, sdkerrors.Wrapf(govtypes.ErrInvalidSigner, "invalid authority; expected %s, got %s", k.authority, req.Metadata.Creator)
+ }
+ return &emptypb.Empty{}, k.overrideNonce(ctx, req.ChainReferenceId, req.Nonce)
+}

x/skyway/keeper/msg_server_test.go

@@ -3,13 +3,15 @@ package keeper
 import (
  "crypto/ecdsa"
  "encoding/hex"
+ "fmt"
  "math/rand"
  "strings"
  "testing"
  "unicode"
 
  sdk "github.com/cosmos/cosmos-sdk/types"
  "github.com/palomachain/paloma/x/skyway/types"
+ valsettypes "github.com/palomachain/paloma/x/valset/types"
  "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/require"
 )
@@ -106,3 +108,49 @@ func TestConfirmHandlerCommonWithMixedCaseAddress(t *testing.T) {
  ret_err := confirmHandlerCommonWithAddress(t, string(mixedCase), initVar)
  assert.Nil(t, ret_err)
 }
+
+func TestOverrideNonceProposal(t *testing.T) {
+ input, ctx := SetupFiveValChain(t)
+ defer func() {
+ sdk.UnwrapSDKContext(ctx).Logger().Info("Asserting invariants at test end")
+ input.AssertInvariants()
+ }()
+
+ proposal := types.MsgNonceOverrideProposal{
+ Metadata: valsettypes.MsgMetadata{
+ Creator: input.SkywayKeeper.authority,
+ Signers: []string{input.SkywayKeeper.authority},
+ },
+ ChainReferenceId: "test-chain",
+ Nonce: 3,
+ }
+
+ err := input.SkywayKeeper.setLastObservedSkywayNonce(ctx, "test-chain", 15)
+ require.NoError(t, err)
+
+ lastObserved, err := input.SkywayKeeper.GetLastObservedSkywayNonce(ctx, "test-chain")
+ require.NoError(t, err)
+ require.Equal(t, uint64(15), lastObserved)
+
+ for k, v := range map[int]uint64{1: 15, 2: 14, 3: 13} {
+ err = input.SkywayKeeper.SetLastSkywayNonceByValidator(ctx, sdk.ValAddress(fmt.Sprintf("validator-%d", k)), "test-chain", v)
+ require.NoError(t, err)
+ lastObserved, err = input.SkywayKeeper.GetLastSkywayNonceByValidator(ctx, sdk.ValAddress(fmt.Sprintf("validator-%d", k)), "test-chain")
+ require.NoError(t, err)
+ require.Equal(t, v, lastObserved)
+ }
+
+ sv := msgServer{input.SkywayKeeper}
+ _, err = sv.OverrideNonceProposal(ctx, &proposal)
+ require.NoError(t, err)
+
+ lastObserved, err = input.SkywayKeeper.GetLastObservedSkywayNonce(ctx, "test-chain")
+ require.NoError(t, err)
+ require.Equal(t, proposal.Nonce, lastObserved)
+
+ for k := range map[int]uint64{1: 15, 2: 14, 3: 13} {
+ lastObserved, err = input.SkywayKeeper.GetLastSkywayNonceByValidator(ctx, sdk.ValAddress(fmt.Sprintf("validator-%d", k)), "test-chain")
+ require.NoError(t, err)
+ require.Equal(t, proposal.Nonce, lastObserved, "failed with validator %d", k)
+ }
+}

x/skyway/types/codec.go

@@ -30,6 +30,7 @@ func RegisterInterfaces(registry types.InterfaceRegistry) {
  &MsgCancelSendToRemote{},
  &MsgSubmitBadSignatureEvidence{},
  &MsgLightNodeSaleClaim{},
+ &MsgNonceOverrideProposal{},
  )
 
  registry.RegisterInterface(
@@ -83,4 +84,5 @@ func RegisterCodec(cdc *codec.LegacyAmino) {
  cdc.RegisterConcrete(&SetBridgeTransferLimitProposal{}, "skyway/SetBridgeTransferLimitProposal", nil)
  cdc.RegisterConcrete(&MsgLightNodeSaleClaim{}, "skyway/MsgLightNodeSaleClaim", nil)
  cdc.RegisterConcrete(&SetLightNodeSaleContractsProposal{}, "skyway/SetLightNodeSaleContractsProposal", nil)
+ cdc.RegisterConcrete(&MsgNonceOverrideProposal{}, "skyway/MsgNonceOverrideProposal", nil)
 }