Skip to content

Commit

Permalink
fix(ci): add timeout for snapshot downloads
Browse files Browse the repository at this point in the history
  • Loading branch information
0xNineteen committed Jan 20, 2025
1 parent 63544d0 commit da8ef17
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 0 deletions.
11 changes: 11 additions & 0 deletions src/accountsdb/download.zig
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ pub fn downloadSnapshotsFromGossip(
output_dir: std.fs.Dir,
min_mb_per_sec: usize,
max_number_of_download_attempts: u64,
timeout: ?sig.time.Duration,
) !struct { std.fs.File, ?std.fs.File } {
const logger = logger_.withScope(LOG_SCOPE);
logger
Expand All @@ -196,6 +197,7 @@ pub fn downloadSnapshotsFromGossip(
var slow_peer_pubkeys = std.ArrayList(Pubkey).init(allocator);
defer slow_peer_pubkeys.deinit();

var function_duration = try std.time.Timer.start();
var download_attempts: u64 = 0;
while (true) {
std.time.sleep(5 * std.time.ns_per_s); // wait while gossip table updates
Expand All @@ -205,6 +207,13 @@ pub fn downloadSnapshotsFromGossip(
return error.UnableToDownloadSnapshot;
}

if (timeout) |t| {
if (function_duration.read() > t.asNanos()) {
logger.err().logf("exceeded download timeout: {any}", .{t});
return error.UnableToDownloadSnapshot;
}
}

// only hold gossip table lock for this block
{
const gossip_table, var gossip_table_lg = gossip_service.gossip_table_rw.readWithLock();
Expand Down Expand Up @@ -449,6 +458,7 @@ pub fn getOrDownloadAndUnpackSnapshot(
min_snapshot_download_speed_mbs: usize = 20,
trusted_validators: ?[]const Pubkey = null,
max_number_of_download_attempts: u64 = default_adb_config.max_number_of_snapshot_download_attempts,
download_timeout: ?sig.time.Duration = null,
},
) !struct { FullAndIncrementalManifest, SnapshotFiles } {
const logger = logger_.withScope(LOG_SCOPE);
Expand Down Expand Up @@ -504,6 +514,7 @@ pub fn getOrDownloadAndUnpackSnapshot(
snapshot_dir,
@intCast(min_mb_per_sec),
options.max_number_of_download_attempts,
options.download_timeout,
);
defer full.close();
defer if (maybe_inc) |inc| inc.close();
Expand Down
1 change: 1 addition & 0 deletions src/benchmarks.zig
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ pub fn main() !void {
.force_new_snapshot_download = true,
.max_number_of_download_attempts = 50,
.min_snapshot_download_speed_mbs = 10,
.download_timeout = Duration.fromMinutes(5),
},
) catch |err| {
switch (err) {
Expand Down
1 change: 1 addition & 0 deletions src/cmd/cmd.zig
Original file line number Diff line number Diff line change
Expand Up @@ -1554,6 +1554,7 @@ fn downloadSnapshot() !void {
snapshot_dir,
@intCast(min_mb_per_sec),
config.current.accounts_db.max_number_of_snapshot_download_attempts,
null,
);
defer full_file.close();
defer if (maybe_inc_file) |inc_file| inc_file.close();
Expand Down
4 changes: 4 additions & 0 deletions src/time/time.zig
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,10 @@ pub const Duration = struct {
return .{ .ns = 0 };
}

pub fn fromMinutes(m: u64) Duration {
return .{ .ns = m * 60 * std.time.ns_per_s };
}

pub fn fromSecs(s: u64) Duration {
return .{ .ns = s * std.time.ns_per_s };
}
Expand Down

0 comments on commit da8ef17

Please sign in to comment.