feat(consensus)!: suspend nodes that fail to participate (#1181)

Description
---
- suspend nodes that fail to participate in 3 rounds (configurable)
- resume nodes after voting in 5 rounds 
- refactor and simplify leader failure

Motivation and Context
---
Reduce the impact of a nodes going offline by enabling nodes to agree to
suspend the offending nodes from consensus rounds. Once suspended, nodes
will immediately send a newview to the next functioning node.

How Has This Been Tested?
---
Unit test, manually 5 nodes taking one offline, observe suspension, go
online, wait for resumed node instruction

What process can a PR reviewer use to test or verify this change?
---
As above

Breaking Changes
---

- [ ] None
- [x] Requires data directory to be deleted
- [x] Other - Please specify

BREAKING CHANGE: 2 new block commands added

.github/workflows/ci.yml

@@ -248,7 +248,7 @@ jobs:
           path: ${{ github.workspace }}/target/nextest/ci/junit.xml
 
       - name: cargo test cucumber
-        run: cargo test --all-features --release --package integration_tests
+        run: cargo test --release --package integration_tests --test cucumber -- --tags "not @ignore and not @flaky"
 
       - name: upload test result artifact
         uses: actions/upload-artifact@v4 # upload test results as artifact

applications/tari_swarm_daemon/webui/src/routes/Main.tsx

@@ -52,7 +52,7 @@ function ExtraInfoVN({name, url, addTxToPool, autoRefresh, state, horizontal}: {
     const [epoch, setEpoch] = useState(null);
     const [height, setHeight] = useState(null);
     const [pool, setPool] = useState([]);
-    const [copied, setCopied] = useState(false);
+    const [copied, setCopied] = useState<string | null>(null);
     const [missingTxStates, setMissingTxStates] = useState({}); // {tx_id: [vn1, vn2, ...]}
     const [publicKey, setPublicKey] = useState(null);
     const [peerId, setPeerId] = useState(null);
@@ -128,11 +128,11 @@ function ExtraInfoVN({name, url, addTxToPool, autoRefresh, state, horizontal}: {
     };
     useEffect(() => {
         if (copied) {
-            setTimeout(() => setCopied(false), 1000);
+            setTimeout(() => setCopied(null), 1000);
         }
     }, [copied]);
     const copyToClipboard = (str: string) => {
-        setCopied(true);
+        setCopied(str);
         navigator.clipboard.writeText(str);
     };
     const showMissingTx = (missingTxStates: { [key: string]: any }) => {
@@ -157,7 +157,7 @@ function ExtraInfoVN({name, url, addTxToPool, autoRefresh, state, horizontal}: {
                         const {known, abort_details, final_decision} = missingTxStates[tx];
                         return (
                             <>
-                                <div onClick={() => copyToClipboard(tx)}>{copied && "Copied" || shorten(tx)}</div>
+                                <div onClick={() => copyToClipboard(tx)}>{copied == tx ? "Copied" : shorten(tx)}</div>
                                 <div style={{color: known ? "green" : "red"}}><b>{known && "Yes" || "No"}</b></div>
                                 <div>{abort_details || <i>unknown</i>}</div>
                                 <div>{final_decision || <i>unknown</i>}</div>

applications/tari_validator_node/src/consensus/leader_selection.rs

@@ -14,6 +14,6 @@ impl RoundRobinLeaderStrategy {
 
 impl<TAddr: NodeAddressable> LeaderStrategy<TAddr> for RoundRobinLeaderStrategy {
     fn calculate_leader(&self, committee: &Committee<TAddr>, height: NodeHeight) -> u32 {
-        (height.0 % committee.members.len() as u64) as u32
+        (height.as_u64() % committee.members.len() as u64) as u32
     }
 }

applications/tari_validator_node/src/p2p/services/mempool/gossip.rs

@@ -155,6 +155,10 @@ impl MempoolGossip<PeerAddress> {
             ))
             .filter(|sg| exclude_shard_group.as_ref() != Some(sg) && sg != &local_shard_group)
             .collect::<HashSet<_>>();
+        // If the only shard group involved is the excluded one.
+        if shard_groups.is_empty() {
+            return Ok(());
+        }
 
         let msg = self
             .codec

applications/tari_validator_node_web_ui/src/routes/Blocks/BlockDetails.tsx

@@ -53,6 +53,12 @@ const COMMANDS = [
   "AllAccept",
   "SomeAccept",
 ];
+
+type OtherCommands = Record<string, Array<any>>;
+// interface OtherCommands {
+//   [key: string]: Array<any>;
+// }
+
 export default function BlockDetails() {
   const { blockId } = useParams();
   const [expandedPanels, setExpandedPanels] = useState<string[]>([]);
@@ -61,6 +67,7 @@ export default function BlockDetails() {
   const [block, setBlock] = useState<Block>();
 
   const [blockData, setBlockData] = useState<{ [key: string]: TransactionAtom[] }>({});
+  const [otherCommands, setOtherCommands] = useState<OtherCommands>({});
 
   const [epochEvents, setEpochEvents] = useState<string[]>([]);
   const [identity, setIdentity] = useState<VNGetIdentityResponse>();
@@ -84,6 +91,7 @@ export default function BlockDetails() {
             });
           }
           setEpochEvents([]);
+          const otherCommands: OtherCommands = {};
           const foreignProposals = [];
           const mintedUtxos = [];
           const data: { [key: string]: TransactionAtom[] } = {};
@@ -92,13 +100,20 @@ export default function BlockDetails() {
 
               const cmd = Object.keys(command)[0];
 
-              if ("ForeignProposal" in command) {
+              if (COMMANDS.indexOf(cmd) > -1) {
+                data[cmd] ||= [];
+                data[cmd].push(command[cmd as keyof Command]);
+              } else if ("ForeignProposal" in command) {
                 foreignProposals.push(command.ForeignProposal);
               } else if ("MintConfidentialOutput" in command) {
                 mintedUtxos.push(command.MintConfidentialOutput);
               } else {
-                data[cmd] ||= [];
-                data[cmd].push(command[cmd as keyof Command]);
+                if (Array.isArray(otherCommands[cmd])) {
+                  otherCommands[cmd].push(command[cmd as keyof Command]);
+                } else {
+                  // command[cmd as keyof Command]});
+                  Object.assign(otherCommands, { [cmd]: [command[cmd as keyof Command]] });
+                }
               }
             } else {
               setEpochEvents((epochEvents: string[]) => [...epochEvents, command as string]);
@@ -108,6 +123,8 @@ export default function BlockDetails() {
           setForeignProposals(foreignProposals);
           setMintedUtxos(mintedUtxos);
           setBlockData(data);
+          setOtherCommands(otherCommands);
+
         })
         .catch((err) => {
           setError(err && err.message ? err.message : `Unknown error: ${JSON.stringify(err)}`);
@@ -314,6 +331,17 @@ export default function BlockDetails() {
                     </AccordionDetails>
                   </Accordion>
                 )}
+                {Object.keys(otherCommands).length > 0 && Object.keys(otherCommands).map((key, i) => (
+                  <Accordion key={i} expanded={expandedPanels.includes(`panel${key}`)}
+                             onChange={handleChange(`panel${key}`)}>
+                    <AccordionSummary aria-controls={`panel${key}bh-content`} id={`panel${key}sbh-header`}>
+                      <Typography>{key}</Typography>
+                    </AccordionSummary>
+                    <AccordionDetails>
+                      <ul>{otherCommands[key].map((elem, j) => <li key={j}>{JSON.stringify(elem)}</li>)}</ul>
+                    </AccordionDetails>
+                  </Accordion>
+                ))}
               </div>
             </Fade>
           )}

bindings/dist/index.d.ts

@@ -2,24 +2,24 @@ export * from "./types/AbortReason";
 export * from "./types/AccessRule";
 export * from "./types/Account";
 export * from "./types/Amount";
-export * from "./types/Arg";
 export * from "./types/ArgDef";
+export * from "./types/Arg";
 export * from "./types/AuthHook";
 export * from "./types/Block";
 export * from "./types/BucketId";
 export * from "./types/Claims";
 export * from "./types/Command";
-export * from "./types/Committee";
 export * from "./types/CommitteeInfo";
 export * from "./types/CommitteeShardInfo";
+export * from "./types/Committee";
 export * from "./types/ComponentAccessRules";
 export * from "./types/ComponentAddress";
 export * from "./types/ComponentBody";
 export * from "./types/ComponentHeader";
 export * from "./types/ComponentKey";
 export * from "./types/ConfidentialClaim";
-export * from "./types/ConfidentialOutput";
 export * from "./types/ConfidentialOutputStatement";
+export * from "./types/ConfidentialOutput";
 export * from "./types/ConfidentialStatement";
 export * from "./types/ConfidentialTransferInputSelection";
 export * from "./types/ConfidentialWithdrawProof";
@@ -29,12 +29,12 @@ export * from "./types/EntityId";
 export * from "./types/Epoch";
 export * from "./types/Event";
 export * from "./types/Evidence";
-export * from "./types/ExecuteResult";
 export * from "./types/ExecutedTransaction";
+export * from "./types/ExecuteResult";
 export * from "./types/ExtraData";
 export * from "./types/FeeBreakdown";
-export * from "./types/FeeClaim";
 export * from "./types/FeeClaimAddress";
+export * from "./types/FeeClaim";
 export * from "./types/FeeCostBreakdown";
 export * from "./types/FeeReceipt";
 export * from "./types/FeeSource";
@@ -43,10 +43,10 @@ export * from "./types/ForeignProposalAtom";
 export * from "./types/FunctionDef";
 export * from "./types/IndexedValue";
 export * from "./types/IndexedWellKnownTypes";
-export * from "./types/Instruction";
 export * from "./types/InstructionResult";
-export * from "./types/JrpcPermission";
+export * from "./types/Instruction";
 export * from "./types/JrpcPermissions";
+export * from "./types/JrpcPermission";
 export * from "./types/LeaderFee";
 export * from "./types/LockFlag";
 export * from "./types/LogEntry";
@@ -55,14 +55,14 @@ export * from "./types/Metadata";
 export * from "./types/MintConfidentialOutputAtom";
 export * from "./types/NetworkCommitteeInfo";
 export * from "./types/NodeHeight";
-export * from "./types/NonFungible";
-export * from "./types/NonFungibleAddress";
 export * from "./types/NonFungibleAddressContents";
+export * from "./types/NonFungibleAddress";
 export * from "./types/NonFungibleContainer";
 export * from "./types/NonFungibleId";
-export * from "./types/NonFungibleIndex";
 export * from "./types/NonFungibleIndexAddress";
+export * from "./types/NonFungibleIndex";
 export * from "./types/NonFungibleToken";
+export * from "./types/NonFungible";
 export * from "./types/NumPreshards";
 export * from "./types/Ordering";
 export * from "./types/OwnerRule";
@@ -72,48 +72,50 @@ export * from "./types/QuorumCertificate";
 export * from "./types/QuorumDecision";
 export * from "./types/RejectReason";
 export * from "./types/RequireRule";
-export * from "./types/Resource";
 export * from "./types/ResourceAccessRules";
 export * from "./types/ResourceAddress";
 export * from "./types/ResourceContainer";
+export * from "./types/Resource";
 export * from "./types/ResourceType";
 export * from "./types/RestrictedAccessRule";
+export * from "./types/ResumeNodeAtom";
 export * from "./types/RuleRequirement";
-export * from "./types/Shard";
 export * from "./types/ShardEvidence";
-export * from "./types/ShardGroup";
 export * from "./types/ShardGroupEvidence";
-export * from "./types/Substate";
+export * from "./types/ShardGroup";
+export * from "./types/Shard";
 export * from "./types/SubstateAddress";
 export * from "./types/SubstateDestroyed";
 export * from "./types/SubstateDiff";
 export * from "./types/SubstateId";
 export * from "./types/SubstateLockType";
 export * from "./types/SubstateRecord";
-export * from "./types/SubstateRequirement";
 export * from "./types/SubstateRequirementLockIntent";
+export * from "./types/SubstateRequirement";
+export * from "./types/Substate";
 export * from "./types/SubstateType";
 export * from "./types/SubstateValue";
+export * from "./types/SuspendNodeAtom";
 export * from "./types/TemplateDef";
 export * from "./types/TemplateDefV1";
-export * from "./types/Transaction";
 export * from "./types/TransactionAtom";
 export * from "./types/TransactionPoolRecord";
 export * from "./types/TransactionPoolStage";
-export * from "./types/TransactionReceipt";
 export * from "./types/TransactionReceiptAddress";
+export * from "./types/TransactionReceipt";
 export * from "./types/TransactionResult";
 export * from "./types/TransactionSignature";
 export * from "./types/TransactionStatus";
+export * from "./types/Transaction";
 export * from "./types/Type";
-export * from "./types/UnclaimedConfidentialOutput";
 export * from "./types/UnclaimedConfidentialOutputAddress";
+export * from "./types/UnclaimedConfidentialOutput";
 export * from "./types/UnsignedTransaction";
 export * from "./types/ValidatorSignature";
-export * from "./types/Vault";
 export * from "./types/VaultId";
-export * from "./types/VersionedSubstateId";
+export * from "./types/Vault";
 export * from "./types/VersionedSubstateIdLockIntent";
+export * from "./types/VersionedSubstateId";
 export * from "./types/ViewableBalanceProof";
 export * from "./base-node-client";
 export * from "./tari-indexer-client";