From 82c847a1df3814d6204bbde985df9685dce69bde Mon Sep 17 00:00:00 2001 From: Sunny Gleason Date: Wed, 23 Oct 2019 16:23:55 -0400 Subject: [PATCH] feat: use epoch schedule for uptime calculation --- api/fullnode-url.js | 2 +- api/loaders/tourdesol/index.js | 16 +++- api/uptime-crawler.js | 85 +++++++++++------- api/util.js | 95 +++++++++++++++++++++ api/views/tourdesol/index.js | 33 +++++-- src/v2/components/TourDeSol/Table/index.jsx | 47 ++++++++-- src/v2/components/ValidatorsMap/Marker.jsx | 4 +- 7 files changed, 230 insertions(+), 52 deletions(-) diff --git a/api/fullnode-url.js b/api/fullnode-url.js index bf429e6c..56963783 100644 --- a/api/fullnode-url.js +++ b/api/fullnode-url.js @@ -1,2 +1,2 @@ -//const FULLNODE_URL = 'http://beta.testnet.solana.com:8899'; +// export const FULLNODE_URL = 'http://beta.testnet.solana.com:8899'; export const FULLNODE_URL = process.env.FULLNODE_URL || 'http://localhost:8899'; diff --git a/api/loaders/tourdesol/index.js b/api/loaders/tourdesol/index.js index 6e8fe8ee..fd467b68 100644 --- a/api/loaders/tourdesol/index.js +++ b/api/loaders/tourdesol/index.js @@ -1,4 +1,7 @@ +import * as solanaWeb3 from '@solana/web3.js'; + import {FriendlyGet} from '../../friendlyGet'; +import {FULLNODE_URL} from '../../fullnode-url'; /** * loadTourDeSolIndex: retrieves raw data from the data store and returns it for formatting @@ -7,8 +10,17 @@ import {FriendlyGet} from '../../friendlyGet'; * @returns {Promise<{__errors__: *, clusterInfo: *}>} */ export async function loadTourDeSolIndex(redisX, {isDemo, activeStage}) { - const {__errors__, redisKeys} = await new FriendlyGet() + const connection = new solanaWeb3.Connection(FULLNODE_URL); + + const { + __errors__, + redisKeys, + epochInfo, + epochSchedule, + } = await new FriendlyGet() .with('redisKeys', redisX.mgetAsync('!clusterInfo', '!blk-last-slot')) + .with('epochInfo', connection.getEpochInfo()) + .with('epochSchedule', connection.getEpochSchedule()) .get(); const [clusterInfoJson, lastSlotString] = redisKeys; @@ -21,5 +33,7 @@ export async function loadTourDeSolIndex(redisX, {isDemo, activeStage}) { activeStage, clusterInfo, lastSlot, + epochInfo, + epochSchedule, }; } diff --git a/api/uptime-crawler.js b/api/uptime-crawler.js index ffcfa13d..69746578 100644 --- a/api/uptime-crawler.js +++ b/api/uptime-crawler.js @@ -20,12 +20,52 @@ function getClient() { const client = getClient(); const setAsync = promisify(client.set).bind(client); -// FIXME: this should be a genesis block API call (eventually), see: -// https://github.com/solana-labs/solana/blob/master/cli/src/wallet.rs#L680 -// https://github.com/solana-labs/solana/blob/master/sdk/src/timing.rs#L14 -const SLOTS_PER_EPOCH = 8192; +function getSlotsInEpoch(epochSchedule, epoch) { + if (!epochSchedule.warmup || epoch >= epochSchedule.first_normal_epoch) { + return epochSchedule.slots_per_epoch; + } + + return null; +} + +function getUptime(epochSchedule, voteState, lat, ts) { + const uptime = _.reduce( + voteState.epochCredits, + (a, v) => { + const slotsInEpoch = + epochSchedule && + v && + v.epoch && + getSlotsInEpoch(epochSchedule, v.epoch); + + if (!slotsInEpoch) { + return a; + } + + const credits = v.credits - v.prevCredits; + + a.unshift({ + epoch: v.epoch, + credits_earned: credits, + slots_in_epoch: slotsInEpoch, + percentage: ((credits * 1.0) / (slotsInEpoch * 1.0)).toFixed(6), + }); + + return a; + }, + [], + ); + + return { + nodePubkey: voteState.nodePubkey.toString(), + authorizedVoterPubkey: voteState.authorizedVoterPubkey.toString(), + uptime, + lat, + ts, + }; +} -async function getVoteAccountUptime(connection, x) { +async function getVoteAccountUptime(connection, epochSchedule, x) { const t1 = new Date().getTime(); let {voteAccount} = await new FriendlyGet() .with( @@ -37,32 +77,12 @@ async function getVoteAccountUptime(connection, x) { let voteState = solanaWeb3.VoteAccount.fromAccountData(voteAccount.data); if (voteState) { - const uptime = _.reduce( - voteState.epochCredits, - (a, v) => { - let credits = v.credits - v.prevCredits; - - a.unshift({ - epoch: v.epoch, - credits_earned: credits, - slots_in_epoch: SLOTS_PER_EPOCH, - percentage: ((credits * 1.0) / (SLOTS_PER_EPOCH * 1.0)).toFixed(6), - }); - - return a; - }, - [], + return getUptime( + epochSchedule, + voteState, + t2 - t1, + new Date(t1).toISOString(), ); - - const uptimeValue = { - nodePubkey: voteState.nodePubkey.toString(), - authorizedVoterPubkey: voteState.authorizedVoterPubkey.toString(), - uptime: uptime, - lat: t2 - t1, - ts: t1, - }; - - return uptimeValue; } else { console.log('eep, no vote state: ', x.votePubkey); return null; @@ -73,15 +93,16 @@ async function refreshUptime() { console.log('uptime updater: updating...'); try { const connection = new solanaWeb3.Connection(FULLNODE_URL); - let {__errors__, voting} = await new FriendlyGet() + let {__errors__, voting, epochSchedule} = await new FriendlyGet() .with('voting', connection.getVoteAccounts()) + .with('epochSchedule', connection.getEpochSchedule()) .get(); let allAccounts = (voting && voting.current ? voting.current : []).concat( voting && voting.delinquent ? voting.delinquent : [], ); const resultsAsync = _.map(allAccounts, v => { - return getVoteAccountUptime(connection, v); + return getVoteAccountUptime(connection, epochSchedule, v); }); let results = await Promise.all(resultsAsync); diff --git a/api/util.js b/api/util.js index eff766a3..db17f274 100644 --- a/api/util.js +++ b/api/util.js @@ -5,6 +5,101 @@ const b58e = Base58.encode; export const LAMPORT_SOL_RATIO = 0.0000000000582; +const DEFAULT_CUMULATIVE_UPTIME_EPOCHS = 64; +const TDS_MAGIC_EPOCH = 10; + +export function calculateUptimeValues(epochInfo, epochSchedule, uptimeValues) { + const {epoch} = epochInfo; + const { + first_normal_epoch: firstNormalEpoch, + slots_per_epoch: slotsPerEpoch, + } = epochSchedule; + + const lastEpoch = epoch - 1; + const firstEpoch = Math.max(firstNormalEpoch, TDS_MAGIC_EPOCH); + + if (lastEpoch < firstEpoch) { + return { + lastEpochUptimePercent: null, + lastEpochUptimeCreditsEarned: null, + lastEpochUptimeCreditsPossible: null, + cumulativeUptimeCreditsEarned: null, + cumulativeUptimeCreditsPossible: null, + cumulativeUptimePercent: null, + complete: false, + epochs: 0, + }; + } + + const accumulated = _.reduce( + uptimeValues, + (a, v) => { + const { + lastEpochUptimeCreditsEarned, + cumulativeUptimeCreditsEarned, + epochsSeen, + } = a; + + const {epoch: thisEpoch, credits_earned: creditsEarned} = v; + + if ( + thisEpoch < firstEpoch || + thisEpoch > lastEpoch || + epochsSeen[thisEpoch] + ) { + return a; + } + + epochsSeen[thisEpoch] = true; + + return { + lastEpochUptimeCreditsEarned: + thisEpoch === lastEpoch + ? creditsEarned + : lastEpochUptimeCreditsEarned, + cumulativeUptimeCreditsEarned: + creditsEarned + cumulativeUptimeCreditsEarned, + epochsSeen, + }; + }, + { + lastEpochUptimeCreditsEarned: 0, + cumulativeUptimeCreditsEarned: 0, + epochsSeen: {}, + }, + ); + + const { + lastEpochUptimeCreditsEarned, + cumulativeUptimeCreditsEarned, + epochsSeen, + } = accumulated; + + const lastEpochUptimeCreditsPossible = slotsPerEpoch; + const cumulativeUptimeCreditsPossible = + Math.min(DEFAULT_CUMULATIVE_UPTIME_EPOCHS, lastEpoch - firstEpoch + 1) * + slotsPerEpoch; + + const lastEpochUptimePercent = + (100 * (lastEpochUptimeCreditsEarned * 1.0)) / + (lastEpochUptimeCreditsPossible * 1.0); + const cumulativeUptimePercent = + (100 * (cumulativeUptimeCreditsEarned * 1.0)) / + (cumulativeUptimeCreditsPossible * 1.0); + + return { + lastEpoch, + lastEpochUptimePercent, + lastEpochUptimeCreditsEarned, + lastEpochUptimeCreditsPossible, + cumulativeUptimeCreditsEarned, + cumulativeUptimeCreditsPossible, + cumulativeUptimePercent, + complete: lastEpoch - firstEpoch >= DEFAULT_CUMULATIVE_UPTIME_EPOCHS, + uptimeEpochs: _.size(epochsSeen), + }; +} + export function transactionFromJson(x, outMessage = {}) { let txn = Transaction.from(Buffer.from(x)); let tx = {}; diff --git a/api/views/tourdesol/index.js b/api/views/tourdesol/index.js index 60833fcf..cb84c492 100644 --- a/api/views/tourdesol/index.js +++ b/api/views/tourdesol/index.js @@ -1,6 +1,6 @@ import {filter, reduce, orderBy} from 'lodash/fp'; -import {LAMPORT_SOL_RATIO} from '../../util'; +import {calculateUptimeValues, LAMPORT_SOL_RATIO} from '../../util'; const SLOTS_PER_DAY = (1.0 * 24 * 60 * 60) / 0.8; const TDS_DEFAULT_STAGE_LENGTH_BLOCKS = SLOTS_PER_DAY * 5.0; @@ -49,7 +49,14 @@ export class TourDeSolIndexView { }; } - const {isDemo, activeStage, clusterInfo, lastSlot} = rawData; + const { + isDemo, + activeStage, + clusterInfo, + lastSlot, + epochInfo, + epochSchedule, + } = rawData; const activeValidatorsRaw = clusterInfo && @@ -96,11 +103,18 @@ export class TourDeSolIndexView { const activatedStakePercent = clusterInfo && 100.0 * (x.activatedStake / clusterInfo.totalStaked); - const uptimePercent = - x.uptime && - x.uptime.uptime && - x.uptime.uptime.length && - Math.min(100.0, 100.0 * parseFloat(x.uptime.uptime[0].percentage)); + const uptime = calculateUptimeValues( + epochInfo, + epochSchedule, + x.uptime.uptime, + ); + + const { + lastEpochUptimePercent, + cumulativeUptimePercent, + uptimeEpochs, + } = uptime; + const score = this.computeNodeScore(x, scoreParams); const validator = { @@ -110,7 +124,10 @@ export class TourDeSolIndexView { activatedStake, activatedStakePercent, slot, - uptimePercent, + lastEpochUptimePercent, + cumulativeUptimePercent, + uptimeEpochs, + uptime, score, }; diff --git a/src/v2/components/TourDeSol/Table/index.jsx b/src/v2/components/TourDeSol/Table/index.jsx index 36f480c1..0e63d490 100644 --- a/src/v2/components/TourDeSol/Table/index.jsx +++ b/src/v2/components/TourDeSol/Table/index.jsx @@ -59,7 +59,9 @@ const ValidatorsTable = ({ avatarUrl, activatedStake, activatedStakePercent, - uptimePercent, + lastEpochUptimePercent, + cumulativeUptimePercent, + uptimeEpochs, rank, } = row; @@ -73,9 +75,23 @@ const ValidatorsTable = ({ {activatedStake.toFixed(8) || 'N/A'} ( {activatedStakePercent.toFixed(3)}%) - - {(uptimePercent && - `${uptimePercent.toFixed(uptimePercent ? 4 : 2)}%`) || + + {(cumulativeUptimePercent && + `${cumulativeUptimePercent.toFixed( + cumulativeUptimePercent ? 4 : 2, + )}%`) || 'Unavailable'} @@ -88,7 +104,9 @@ const ValidatorsTable = ({ avatarUrl, activatedStake, activatedStakePercent, - uptimePercent, + lastEpochUptimePercent, + cumulativeUptimePercent, + uptimeEpochs, } = card; return (
Uptime
-
- {(uptimePercent && - `${uptimePercent.toFixed(uptimePercent ? 4 : 2)}%`) || +
+ {(cumulativeUptimePercent && + `${cumulativeUptimePercent.toFixed( + cumulativeUptimePercent ? 4 : 2, + )}%`) || 'Unavailable'}
diff --git a/src/v2/components/ValidatorsMap/Marker.jsx b/src/v2/components/ValidatorsMap/Marker.jsx index 9147d2ee..e2462b2b 100644 --- a/src/v2/components/ValidatorsMap/Marker.jsx +++ b/src/v2/components/ValidatorsMap/Marker.jsx @@ -25,8 +25,8 @@ const Marker = ({scale, marker}: {scale: number, marker: any}) => {