-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: node and protocols health (#2080)
* feat: introduce HealthManager * feat: make health accessible on Waku object * feat: update health from protocols * chore: add access modifiers to healthmanager * feat: use a HealthManager singleton * chore: add tests for Filter, LightPush and Store * feat: add overall node health * chore: update protocol health to consider Store protocol * chore: setup generic test utils instead of using filter utils * tests: add a health status matrix check from 0-3 * chore: increase timeout for failing tests in CI tests pass locally without an increased timeout, but fail in CI * chore: move name inference to HealthManager * tests: abstract away node creation and teardown utils * fix: import
- Loading branch information
1 parent
defe41b
commit d464af3
Showing
21 changed files
with
557 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
import { | ||
HealthStatus, | ||
type IHealthManager, | ||
NodeHealth, | ||
type ProtocolHealth, | ||
Protocols | ||
} from "@waku/interfaces"; | ||
|
||
class HealthManager implements IHealthManager { | ||
public static instance: HealthManager; | ||
private readonly health: NodeHealth; | ||
|
||
private constructor() { | ||
this.health = { | ||
overallStatus: HealthStatus.Unhealthy, | ||
protocolStatuses: new Map() | ||
}; | ||
} | ||
|
||
public static getInstance(): HealthManager { | ||
if (!HealthManager.instance) { | ||
HealthManager.instance = new HealthManager(); | ||
} | ||
return HealthManager.instance; | ||
} | ||
|
||
public getHealthStatus(): HealthStatus { | ||
return this.health.overallStatus; | ||
} | ||
|
||
public getProtocolStatus(protocol: Protocols): ProtocolHealth | undefined { | ||
return this.health.protocolStatuses.get(protocol); | ||
} | ||
|
||
public updateProtocolHealth( | ||
multicodec: string, | ||
connectedPeers: number | ||
): void { | ||
const protocol = this.getNameFromMulticodec(multicodec); | ||
|
||
let status: HealthStatus = HealthStatus.Unhealthy; | ||
if (connectedPeers == 1) { | ||
status = HealthStatus.MinimallyHealthy; | ||
} else if (connectedPeers >= 2) { | ||
status = HealthStatus.SufficientlyHealthy; | ||
} | ||
|
||
this.health.protocolStatuses.set(protocol, { | ||
name: protocol, | ||
status: status, | ||
lastUpdate: new Date() | ||
}); | ||
|
||
this.updateOverallHealth(); | ||
} | ||
|
||
private getNameFromMulticodec(multicodec: string): Protocols { | ||
let name: Protocols; | ||
if (multicodec.includes("filter")) { | ||
name = Protocols.Filter; | ||
} else if (multicodec.includes("lightpush")) { | ||
name = Protocols.LightPush; | ||
} else if (multicodec.includes("store")) { | ||
name = Protocols.Store; | ||
} else { | ||
throw new Error(`Unknown protocol: ${multicodec}`); | ||
} | ||
return name; | ||
} | ||
|
||
private updateOverallHealth(): void { | ||
const relevantProtocols = [Protocols.LightPush, Protocols.Filter]; | ||
const statuses = relevantProtocols.map( | ||
(p) => this.getProtocolStatus(p)?.status | ||
); | ||
|
||
if (statuses.some((status) => status === HealthStatus.Unhealthy)) { | ||
this.health.overallStatus = HealthStatus.Unhealthy; | ||
} else if ( | ||
statuses.some((status) => status === HealthStatus.MinimallyHealthy) | ||
) { | ||
this.health.overallStatus = HealthStatus.MinimallyHealthy; | ||
} else { | ||
this.health.overallStatus = HealthStatus.SufficientlyHealthy; | ||
} | ||
} | ||
} | ||
|
||
export const getHealthManager = (): HealthManager => | ||
HealthManager.getInstance(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import { Protocols } from "./protocols"; | ||
|
||
export enum HealthStatus { | ||
Unhealthy = "Unhealthy", | ||
MinimallyHealthy = "MinimallyHealthy", | ||
SufficientlyHealthy = "SufficientlyHealthy" | ||
} | ||
|
||
export interface IHealthManager { | ||
getHealthStatus: () => HealthStatus; | ||
getProtocolStatus: (protocol: Protocols) => ProtocolHealth | undefined; | ||
updateProtocolHealth: (multicodec: string, connectedPeers: number) => void; | ||
} | ||
|
||
export type NodeHealth = { | ||
overallStatus: HealthStatus; | ||
protocolStatuses: ProtocolsHealthStatus; | ||
}; | ||
|
||
export type ProtocolHealth = { | ||
name: Protocols; | ||
status: HealthStatus; | ||
lastUpdate: Date; | ||
}; | ||
|
||
export type ProtocolsHealthStatus = Map<Protocols, ProtocolHealth>; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
import { waitForRemotePeer } from "@waku/core"; | ||
import { | ||
LightNode, | ||
ProtocolCreateOptions, | ||
Protocols, | ||
ShardingParams, | ||
Waku | ||
} from "@waku/interfaces"; | ||
import { createLightNode } from "@waku/sdk"; | ||
import { isDefined, shardInfoToPubsubTopics } from "@waku/utils"; | ||
import { Context } from "mocha"; | ||
import pRetry from "p-retry"; | ||
|
||
import { DefaultTestPubsubTopic, NOISE_KEY_1 } from "../constants"; | ||
import { ServiceNodesFleet } from "../lib"; | ||
import { Args } from "../types"; | ||
|
||
import { waitForConnections } from "./waitForConnections"; | ||
|
||
export async function runMultipleNodes( | ||
context: Context, | ||
shardInfo?: ShardingParams, | ||
customArgs?: Args, | ||
strictChecking: boolean = false, | ||
numServiceNodes = 3, | ||
withoutFilter = false | ||
): Promise<[ServiceNodesFleet, LightNode]> { | ||
const pubsubTopics = shardInfo | ||
? shardInfoToPubsubTopics(shardInfo) | ||
: [DefaultTestPubsubTopic]; | ||
// create numServiceNodes nodes | ||
const serviceNodes = await ServiceNodesFleet.createAndRun( | ||
context, | ||
pubsubTopics, | ||
numServiceNodes, | ||
strictChecking, | ||
shardInfo, | ||
customArgs, | ||
withoutFilter | ||
); | ||
|
||
const wakuOptions: ProtocolCreateOptions = { | ||
staticNoiseKey: NOISE_KEY_1, | ||
libp2p: { | ||
addresses: { listen: ["/ip4/0.0.0.0/tcp/0/ws"] } | ||
} | ||
}; | ||
|
||
if (shardInfo) { | ||
wakuOptions.shardInfo = shardInfo; | ||
} else { | ||
wakuOptions.pubsubTopics = pubsubTopics; | ||
} | ||
|
||
const waku = await createLightNode(wakuOptions); | ||
await waku.start(); | ||
|
||
if (!waku) { | ||
throw new Error("Failed to initialize waku"); | ||
} | ||
|
||
for (const node of serviceNodes.nodes) { | ||
await waku.dial(await node.getMultiaddrWithId()); | ||
await waitForRemotePeer( | ||
waku, | ||
[ | ||
!customArgs?.filter ? undefined : Protocols.Filter, | ||
!customArgs?.lightpush ? undefined : Protocols.LightPush | ||
].filter(isDefined) | ||
); | ||
await node.ensureSubscriptions(pubsubTopics); | ||
|
||
const wakuConnections = waku.libp2p.getConnections(); | ||
const nodePeers = await node.peers(); | ||
|
||
if (wakuConnections.length < 1 || nodePeers.length < 1) { | ||
throw new Error( | ||
`Expected at least 1 peer in each node. Got waku connections: ${wakuConnections.length} and service nodes: ${nodePeers.length}` | ||
); | ||
} | ||
} | ||
|
||
await waitForConnections(numServiceNodes, waku); | ||
|
||
return [serviceNodes, waku]; | ||
} | ||
|
||
export async function teardownNodesWithRedundancy( | ||
serviceNodes: ServiceNodesFleet, | ||
wakuNodes: Waku | Waku[] | ||
): Promise<void> { | ||
const wNodes = Array.isArray(wakuNodes) ? wakuNodes : [wakuNodes]; | ||
|
||
const stopNwakuNodes = serviceNodes.nodes.map(async (node) => { | ||
await pRetry( | ||
async () => { | ||
await node.stop(); | ||
}, | ||
{ retries: 3 } | ||
); | ||
}); | ||
|
||
const stopWakuNodes = wNodes.map(async (waku) => { | ||
if (waku) { | ||
await pRetry( | ||
async () => { | ||
await waku.stop(); | ||
}, | ||
{ retries: 3 } | ||
); | ||
} | ||
}); | ||
|
||
await Promise.all([...stopNwakuNodes, ...stopWakuNodes]); | ||
} |
Oops, something went wrong.