From 99c1198be00ba2ad435571c20e4fcc9ce665ab2b Mon Sep 17 00:00:00 2001 From: "Jason J. Gullickson" Date: Thu, 25 Apr 2024 08:43:24 -0500 Subject: [PATCH 1/5] It works! --- lib/utils.js | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 237d423..660735d 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -91,13 +91,12 @@ module.exports.load_inode = function load_inode(uri, callback){ }; module.exports.commit_block_to_disk = function commit_block_to_disk(block, block_object, next_storage_location, callback) { + // if storage locations exist, save the block to disk var total_locations = config.STORAGE_LOCATIONS.length; - if(total_locations > 0){ // check all storage locations to see if we already have this block - var on_complete = function on_complete(found_block){ // TODO: consider increasing found count to enable block redundancy if(!found_block){ @@ -107,16 +106,29 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block var dir = config.STORAGE_LOCATIONS[next_storage_location].path; operations.write(dir + block_object.block_hash, block, "binary", function(err){ if (err) { - return callback(err); - } + // If we can't write the block, try other locations + log.message(log.INFO, "Error writing block to " + next_storage_location +", trying next location"); + + // increment (or reset) storage location (striping) + next_storage_location++; + if(next_storage_location === config.STORAGE_LOCATIONS.length){ + next_storage_location = 0; + } + + // TODO: If we've exausted all locations, return the error, + // otherwise this creates an infinate loop. + var dir = config.STORAGE_LOCATIONS[next_storage_location].path; + operations.write(dir + block_object.block_hash, block, "binary", function(err){ + if(err){ + return callback(err); + } + }); + } block_object.last_seen = dir; log.message(log.INFO, "New block " + block_object.block_hash + " written to " + dir); - return callback(null, block_object); - }); - } else { log.message(log.INFO, "Duplicate block " + block_object.block_hash + " not written to disk"); return callback(null, block_object); @@ -127,20 +139,16 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block var location = config.STORAGE_LOCATIONS[idx]; var file = location.path + block_object.block_hash; idx++; - operations.exists(file + ".gz", function(err, result){ - if (result) { log.message(log.INFO, "Duplicate compressed block " + block_object.block_hash + " found in " + location.path); block_object.last_seen = location.path; return on_complete(true); } else { operations.exists(file, function(err_2, result_2){ - if (err_2) { log.message(log.INFO, "Block " + block_object.block_hash + " not found in " + location.path); } - if (result_2) { log.message(log.INFO, "Duplicate block " + block_object.block_hash + " found in " + location.path); block_object.last_seen = location.path; From a77c6e8f031a7c40823388cbf9a89be9301d94e0 Mon Sep 17 00:00:00 2001 From: "Jason J. Gullickson" Date: Thu, 25 Apr 2024 08:56:46 -0500 Subject: [PATCH 2/5] Logging and fixing old bugs. --- lib/utils.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 660735d..1137859 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -105,10 +105,14 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block // TODO: consider implementing in-band compression here var dir = config.STORAGE_LOCATIONS[next_storage_location].path; operations.write(dir + block_object.block_hash, block, "binary", function(err){ + + // I think this fixes an existing error (var dir undefined)? + var dir = config.STORAGE_LOCATIONS[next_storage_location].path; + if (err) { // If we can't write the block, try other locations - log.message(log.INFO, "Error writing block to " + next_storage_location +", trying next location"); + var current_storage_dir = dir // increment (or reset) storage location (striping) next_storage_location++; @@ -116,9 +120,11 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block next_storage_location = 0; } + var dir = config.STORAGE_LOCATIONS[next_storage_location].path; + log.message(log.INFO, "Error writing block to " + current_storage_dir +", trying " + dir); + // TODO: If we've exausted all locations, return the error, // otherwise this creates an infinate loop. - var dir = config.STORAGE_LOCATIONS[next_storage_location].path; operations.write(dir + block_object.block_hash, block, "binary", function(err){ if(err){ return callback(err); From 2bb20a227e156ce93f6e276813759c2a7aae15d9 Mon Sep 17 00:00:00 2001 From: "Jason J. Gullickson" Date: Thu, 25 Apr 2024 08:57:16 -0500 Subject: [PATCH 3/5] cleanup --- lib/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 1137859..211e31c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -106,13 +106,13 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block var dir = config.STORAGE_LOCATIONS[next_storage_location].path; operations.write(dir + block_object.block_hash, block, "binary", function(err){ - // I think this fixes an existing error (var dir undefined)? + // I think this fixes an existing bug (var dir undefined)? var dir = config.STORAGE_LOCATIONS[next_storage_location].path; if (err) { // If we can't write the block, try other locations - var current_storage_dir = dir + var current_storage_dir = dir; // increment (or reset) storage location (striping) next_storage_location++; From 4458c44bf3be5cd33fd33ccfd6c975b8dd6f28c8 Mon Sep 17 00:00:00 2001 From: "Jason J. Gullickson" Date: Thu, 25 Apr 2024 15:58:35 -0500 Subject: [PATCH 4/5] Add escape for endless leveling loop. --- lib/utils.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 211e31c..78a32c5 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -103,17 +103,28 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block // write new block to next storage location // TODO: consider implementing in-band compression here + // Keep track of how many storage locations we try when writes fail + var block_write_retries_remaining = config.STORAGE_LOCATIONS.length; var dir = config.STORAGE_LOCATIONS[next_storage_location].path; operations.write(dir + block_object.block_hash, block, "binary", function(err){ - // I think this fixes an existing bug (var dir undefined)? + // TODO: Figure out why dir goes out of scope so we don't have to do this dumb reassignment var dir = config.STORAGE_LOCATIONS[next_storage_location].path; - if (err) { // If we can't write the block, try other locations var current_storage_dir = dir; + // Give-up if we've tried all the storage locations + // NOTE: I don't think this will ever get tripped, + // because the request will fail if the inode cannot + // get written anywhere and that check kicks-in first + block_write_retries_remaining--; + if(block_write_retries_remaining < 0){ + log.message(log.ERROR, "Unable to write block to any storage locations!"); + return callback(err); + } + // increment (or reset) storage location (striping) next_storage_location++; if(next_storage_location === config.STORAGE_LOCATIONS.length){ @@ -123,8 +134,6 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block var dir = config.STORAGE_LOCATIONS[next_storage_location].path; log.message(log.INFO, "Error writing block to " + current_storage_dir +", trying " + dir); - // TODO: If we've exausted all locations, return the error, - // otherwise this creates an infinate loop. operations.write(dir + block_object.block_hash, block, "binary", function(err){ if(err){ return callback(err); From 8ef3abcb2c62eece4766892866954c26e1a2e196 Mon Sep 17 00:00:00 2001 From: "Jason J. Gullickson" Date: Sat, 11 May 2024 10:40:21 -0500 Subject: [PATCH 5/5] wip --- lib/inode.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/inode.js b/lib/inode.js index 041af2f..1c5577f 100644 --- a/lib/inode.js +++ b/lib/inode.js @@ -122,7 +122,8 @@ var Inode = { // generate a hash of the block to use as a handle/filename block_object.block_hash = utils.sha1_to_hex(block); - utils.commit_block_to_disk(block, block_object, next_storage_location, function(err, result){ + var retries = config.STORAGE_LOCATIONS.length; + utils.commit_block_to_disk(block, block_object, next_storage_location, retries, function(err, result){ if (err) { return callback(err); }