Basic Storage Location Autoleveling

lib/inode.js

@@ -122,7 +122,8 @@ var Inode = {
     // generate a hash of the block to use as a handle/filename
     block_object.block_hash = utils.sha1_to_hex(block);
 
-    utils.commit_block_to_disk(block, block_object, next_storage_location, function(err, result){
+    var retries = config.STORAGE_LOCATIONS.length;
+    utils.commit_block_to_disk(block, block_object, next_storage_location, retries, function(err, result){
       if (err) {
         return callback(err);
       }

lib/utils.js

@@ -91,32 +91,59 @@ module.exports.load_inode = function load_inode(uri, callback){
 };
 
 module.exports.commit_block_to_disk = function commit_block_to_disk(block, block_object, next_storage_location, callback) {
+
   // if storage locations exist, save the block to disk
   var total_locations = config.STORAGE_LOCATIONS.length;
-
   if(total_locations > 0){
 
     // check all storage locations to see if we already have this block
-
     var on_complete = function on_complete(found_block){
       // TODO: consider increasing found count to enable block redundancy
       if(!found_block){
 
         // write new block to next storage location
         // TODO: consider implementing in-band compression here
+        // Keep track of how many storage locations we try when writes fail
+        var block_write_retries_remaining = config.STORAGE_LOCATIONS.length; 
         var dir = config.STORAGE_LOCATIONS[next_storage_location].path;
         operations.write(dir + block_object.block_hash, block, "binary", function(err){
+
+          // TODO: Figure out why dir goes out of scope so we don't have to do this dumb reassignment
+          var dir = config.STORAGE_LOCATIONS[next_storage_location].path;
           if (err) {
-            return callback(err);
-          }
 
+            // If we can't write the block, try other locations
+            var current_storage_dir = dir; 
+
+            // Give-up if we've tried all the storage locations
+            // NOTE: I don't think this will ever get tripped, 
+            // because the request will fail if the inode cannot
+            // get written anywhere and that check kicks-in first
+            block_write_retries_remaining--;
+            if(block_write_retries_remaining < 0){
+              log.message(log.ERROR, "Unable to write block to any storage locations!");
+              return callback(err);
+            }
+
+            // increment (or reset) storage location (striping)
+            next_storage_location++;
+            if(next_storage_location === config.STORAGE_LOCATIONS.length){
+              next_storage_location = 0;
+            }
+
+            var dir = config.STORAGE_LOCATIONS[next_storage_location].path;
+            log.message(log.INFO, "Error writing block to " + current_storage_dir +", trying " + dir);
+
+            operations.write(dir + block_object.block_hash, block, "binary", function(err){
+              if(err){
+                return callback(err);
+              }
+            });
+          }
           block_object.last_seen = dir;
           log.message(log.INFO, "New block " + block_object.block_hash + " written to " + dir);
-
           return callback(null, block_object);
-
         });
-
       } else {
         log.message(log.INFO, "Duplicate block " + block_object.block_hash + " not written to disk");
         return callback(null, block_object);
@@ -127,20 +154,16 @@ module.exports.commit_block_to_disk = function commit_block_to_disk(block, block
       var location = config.STORAGE_LOCATIONS[idx];
       var file = location.path + block_object.block_hash;
       idx++;
-
       operations.exists(file + ".gz", function(err, result){
-
         if (result) {
           log.message(log.INFO, "Duplicate compressed block " + block_object.block_hash + " found in " + location.path);
           block_object.last_seen = location.path;
           return on_complete(true);
         } else {
           operations.exists(file, function(err_2, result_2){
-
             if (err_2) {
               log.message(log.INFO, "Block " + block_object.block_hash + " not found in " + location.path);
             }
-
             if (result_2) {
               log.message(log.INFO, "Duplicate block " + block_object.block_hash + " found in " + location.path);
               block_object.last_seen = location.path;