Merge pull request #3 from DependableSystemsLab/supportCNN

minor changes on the DNN tests

HOWTORUN.md

@@ -63,6 +63,8 @@
 
 2. Follow the instruction to configure the configuration file. By default, it's in the /confFiles/default.yaml file (including the instructions).
 
+3. Run the program and you can observe the output of the ML model under fault.
+
 
 
 **Questions ? Contact Karthik Pattabiraman ([email protected])**

README.md

@@ -14,7 +14,7 @@ details of how TensorFI works, please refer to the Manual file*
 
 ***Updates: 2019-07***
 
-We now support fault injection in complex ML models such as LeNet, AlexNet, as well as support for single bit-flip injection mode. Some DNN models are provided in /Tests directory.
+We now support fault injection in complex ML models such as LeNet, AlexNet, as well as support for single bit-flip injection mode. Some DNN models are provided in /Tests directory. For starter, you can try running the LeNet.py under /Tests/DNN-model/LeNet-mnist/ to inject faults in a CNN (it'll automatically download the dataset and the config file is set up).
 
 You can now create your customized TensorFlow operations for injection, by using the built-in TensorFlow implementation, to support injection on new ML models. See the *Manual* for more details. 
 

Tests/DNN-model/AlexNet-Cifar10/alexNet.py

@@ -267,7 +267,7 @@ def test_and_save(_global_step, epoch):
     print("###########################################################################################################")
 
 
-# train the model
+# Prepreation for training, e.g., setting optimizer
 if(isTrain):
     tf.set_random_seed(21)
 
@@ -307,7 +307,7 @@ def test_and_save(_global_step, epoch):
     train_writer = tf.summary.FileWriter(_SAVE_PATH, sess.graph)
     init = tf.initialize_all_variables()
     sess.run(init)
-
+# Prepreation for testing, e.g., restoring the trained model
 elif(isTest):
     test_x, test_y = get_data_set("test")
     x, y, output, y_pred_cls, global_step, learning_rate = model()

Tests/DNN-model/Dave-steering-model/README.md

@@ -1,4 +1,4 @@
-# Autopilot-TensorFlow
+# Autopilot-TensorFlow (copy from https://github.com/SullyChen/Autopilot-TensorFlow)
 A TensorFlow implementation of this [Nvidia paper](https://arxiv.org/pdf/1604.07316.pdf) with some changes. For a summary of the design process and FAQs, see [this medium article I wrote](https://medium.com/@sullyfchen/how-a-high-school-junior-made-a-self-driving-car-705fa9b6e860).
 
 # IMPORTANT

Tests/DNN-model/Dave-steering-model/run_dataset.py

@@ -22,11 +22,12 @@
 fi = ti.TensorFI(sess, logLevel = 50, name = "convolutional", disableInjections=True)
 
 
-i = 140
+imgIndex = 140
 #while(cv2.waitKey(10) != ord('q')):
 
-for j in range(1):
-    full_image = scipy.misc.imread("driving_dataset/" + str(i) + ".jpg", mode="RGB")
+
+for i in range(1):  # num of imgs to be injected
+    full_image = scipy.misc.imread("driving_dataset/" + str(imgIndex) + ".jpg", mode="RGB")
     image = scipy.misc.imresize(full_image[-150:], [66, 200]) / 255.0
 
     '''    
@@ -52,13 +53,14 @@
 
 
 
-    fi.turnOnInjections()
-    ti.faultTypes.sequentialFIinit()
+    fi.turnOnInjections() 
 
     totalFI = 0.
     sdcCount = 0 
-#    for k in range(3100):
-    while(ti.faultTypes.isKeepDoingFI):
+
+    fiCount = 100
+    for k in range(fiCount): 
+        # steering angle under fault
         degrees = model.y.eval(feed_dict={model.x: [image], model.keep_prob: 1.0})[0][0] * 180.0 / scipy.pi 
 
         totalFI += 1
@@ -88,6 +90,5 @@
 #        dst = cv2.warpAffine(img,M,(cols,rows))
 #        cv2.imshow("steering wheel", dst)
 
-    i += 1 
 
 cv2.destroyAllWindows()

Tests/DNN-model/LeNet-mnist/LeNet.py

@@ -37,6 +37,10 @@
 import tensorflow as tf
 import TensorFI as ti
 
+os.environ["CUDA_VISIBLE_DEVICES"]="-1"  
+
+
+
 SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
 WORK_DIRECTORY = 'data'
 IMAGE_SIZE = 28
@@ -46,7 +50,7 @@
 VALIDATION_SIZE = 500  # Size of the validation set.
 SEED = 66478  # Set to None for random seed.
 BATCH_SIZE = 64
-NUM_EPOCHS = 10
+NUM_EPOCHS = 100
 
 "set to the same size of the test dataset"
 eval_per_batch = 1
@@ -149,10 +153,10 @@ def main(_):
     test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')
 
     # Extract it into numpy arrays.
-    train_data = extract_data(train_data_filename, 10000)
-    train_labels = extract_labels(train_labels_filename, 10000)
+    train_data = extract_data(train_data_filename, 50000)
+    train_labels = extract_labels(train_labels_filename, 50000)
 
-    test_batch = 1
+    test_batch = 100
     test_data = extract_data(test_data_filename, test_batch)
     test_labels = extract_labels(test_labels_filename, test_batch) 
 
@@ -183,21 +187,21 @@ def main(_):
 
 
 
-  depth1 = 2
+  depth1 = 32
   conv1_weights = tf.Variable(
                               tf.truncated_normal([5, 5, NUM_CHANNELS, depth1],  # 5x5 filter, depth 32.
                                                   stddev=0.1,
                                                   seed=SEED, dtype=data_type()))
   conv1_biases = tf.Variable(tf.zeros([depth1], dtype=data_type()))
 
-  depth2 = 2
+  depth2 = 64
   conv2_weights = tf.Variable(
                               tf.truncated_normal([5, 5, depth1, depth2], stddev=0.1,
                               seed=SEED, dtype=data_type()))
   conv2_biases = tf.Variable(tf.constant(0.1, shape=[depth2], dtype=data_type()))
 
 
-  depth3 = 32
+  depth3 = 512
   fc1_weights = tf.Variable(  # fully connected, depth 512.
                             tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * depth2, depth3],
                                                 stddev=0.1,
@@ -340,7 +344,7 @@ def eval_in_batches(data, sess):
       # Run the optimizer to update weights. 
 
       sess.run(optimizer, feed_dict=feed_dict)
-      '''
+
       # print some extra information once reach the evaluation frequency
       if step % EVAL_FREQUENCY == 0:
         # fetch some extra nodes' data
@@ -352,11 +356,14 @@ def eval_in_batches(data, sess):
               (step, float(step) * BATCH_SIZE / train_size,
                1000 * elapsed_time / EVAL_FREQUENCY))
         print('Minibatch loss: %.3f, learning rate: %.6f' % (l, lr))
-        print('Minibatch error: %.1f%%' % error_rate(predictions, batch_labels))
-        print('Validation error: %.1f%%' % error_rate(
-            eval_in_batches(validation_data, sess), validation_labels))
+        errRate, _ = error_rate(predictions, batch_labels)
+        print('Minibatch error: %.1f%%' % errRate )
+#        errRate, _ = error_rate(
+#            eval_in_batches(validation_data, sess), validation_labels)
+#        print('Validation error: %.1f%%' % errRate)
+
         sys.stdout.flush()
-        '''
+
 
 
     # Finally print the result!  
@@ -379,17 +386,14 @@ def eval_in_batches(data, sess):
     # Add the fault injection nodes to it
     fi = ti.TensorFI(sess, logLevel = 50, name = "convolutional", disableInjections=False)
 
-    test_error, indexOfCorrectSample = error_rate(eval_in_batches(test_data, sess), test_labels, True)
-    print('Test error: %.1f%%' % test_error)
-
     # inject ten inputs
     for i in range(10):
       each = indexOfCorrectSample[i]  # index of the sample to be injected
       newData = ( test_data[each].reshape(1,28,28,1) )  # resize the data
       newLab = ( test_labels[each].reshape(1) )
 
 
-      fiCount = 1
+      fiCount = 100   # number of FI
       sdcCount = 0.
       for j in range(fiCount):
         test_error, _ = error_rate(eval_in_batches(newData, sess), newLab, True) 

Tests/DNN-model/LeNet-mnist/confFiles/default.yaml

@@ -0,0 +1,60 @@
+# This is a sample YAML file for fault injection configuration
+# The fields here should correspond to the Fields in fiConfig.py
+
+# Deterministic fault seed for the injections
+# Seed: 1000
+
+# Type of fault to be injected for Scalars and Tensors
+# Allowed values are: {None, Rand, Zero, Rand-element, bitFlip-element, bitFlip-tensor}
+#   1) None - do not inject fault
+#   2) Rand - shuffle all the data items in the output of the target op into random values
+#   3) Zero - change the value into all zeros
+#   4) Rand-element - shuffle one of the data item in the output of the target op into random value
+#   5) bitFlip-element - single bit-flip over one data item in the output of the target op
+#   6) bitFlip-tensor - single bit-flip over all data items in the output of the target op 
+ScalarFaultType: None
+TensorFaultType: bitFlip-element 
+#TensorFaultType: None
+
+# Add the list of Operations and their probabilities here
+# Each entry must be in a separate line ad start with a '-'
+# each line must represent an OP and it's probability value
+# See fiConfig.py for a full list of allowed OP values
+# NOTE: These should not be any tabs anywhere below
+
+Ops:
+# Choose operations which you want to inject
+  - ALL = 1.0 # Chooses all operations
+#  - LRN = 1.0 # Choose one operation only; 
+# - EQUAL = 1.0
+#  - SOFT-MAX = 1.0
+# - DIV = 0.0 # This does not exist - and should be ignored (Test)
+# - SUB = -0.5 # This should raise an exception
+
+# How many times the set of above operations should be skipped before injection
+# SkipCount: 1
+
+
+# Specify the instances of the ML operators 
+# This is required when you want to specify which operator to inject (e.g., the second Conv operator).
+# Below is an example, you need to specify the numbers according to your model.
+Instances: 
+  - CONV2D = 2   # This means there are two conv2d operations in the model
+  - MAX-POOL = 2
+  - RELU = 3
+  - BIASADD = 2
+  - RESHAPE = 1
+  - ADD = 3
+  - MATMUL = 3 
+
+
+
+# There are three inject mode:
+# There are 3 ways to determine HOW to select an operation for injection"
+# 1) Using error rate to determine the probability of each opertaor to be injected " 
+# 2) Profiling the instance of each operation so that each operation will be injected for ONCE only" 
+# 3) Perform random injection over one op per run". In this mode, the 'ALL' variable in the Ops section must be used explicitly. 
+# In the second and third mode, you'll need to specify the instances of all the operators in the algorithm
+
+# Allowed values: 1) "errorRate"; 2) "dynamicInstance"; 3) "oneFaultPerRun" 
+InjectMode: "errorRate"

Tests/DNN-model/README.md

@@ -2,11 +2,11 @@
 
 - This directory contains 6 different DNN models, e.g., steering model in self-driving cars.
 - The benchmarks and datasets are as follows (link to download the dataset): 
-    - LeNet-4 - Mnist dataset (http://yann.lecun.com/exdb/mnist/)
+    - LeNet-4 - Mnist dataset (http://yann.lecun.com/exdb/mnist/). You can directly run this test (it will automatically download the dataset) to try injecting bit-flip in all of the operators. 
     - AlexNet - Cifar-10 (https://www.cs.toronto.edu/~kriz/cifar.html)
     - VGG16 - ImageNet (http://image-net.org/download)
     - VGG11 - German traffic sign (http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset)
     - Comma.ai's steering model - real-world driving frame (https://github.com/SullyChen/driving-datasets)
     - Nvidia Dave steering model - real-world driving frame (https://github.com/SullyChen/driving-datasets) 
 - Note that only vgg16 model is provided with a pre-trained weight (http://www.cs.toronto.edu/~frossard/post/vgg16/); so for the rest of models, you need to train the model first, before performing fault injection.
-- To perform fault injection on your own model, please follow the instruction in the HOWTORUN manual.
+- To perform fault injection on your own model, please follow the instruction in the HOWTORUN manual.

Tests/DNN-model/comma-ai-steering-model/README.md

@@ -1,23 +1 @@
-# Autopilot-TensorFlow
-A TensorFlow implementation of this [Nvidia paper](https://arxiv.org/pdf/1604.07316.pdf) with some changes. For a summary of the design process and FAQs, see [this medium article I wrote](https://medium.com/@sullyfchen/how-a-high-school-junior-made-a-self-driving-car-705fa9b6e860).
-
-# IMPORTANT
-Absolutely, under NO circumstance, should one ever pilot a car using computer vision software trained with this code (or any home made software for that matter). It is extremely dangerous to use your own self-driving software in a car, even if you think you know what you're doing, not to mention it is quite illegal in most places and any accidents will land you in huge lawsuits.
-
-This code is purely for research and statistics, absolutley NOT for application or testing of any sort.
-
-# How to Use
-Download the [dataset](https://github.com/SullyChen/driving-datasets) and extract into the repository folder
-
-Use `python train.py` to train the model
-
-Use `python run.py` to run the model on a live webcam feed
-
-Use `python run_dataset.py` to run the model on the dataset
-
-To visualize training using Tensorboard use `tensorboard --logdir=./logs`, then open http://0.0.0.0:6006/ into your web browser.
-
-# Cited in
-Pan, X., You, Y., Wang, Z., & Lu, C. (2017, September 26). Virtual to Real Reinforcement Learning for Autonomous Driving. Retrieved from [https://arxiv.org/abs/1704.03952](https://arxiv.org/pdf/1704.03952.pdf)
-
-https://medium.com/@maxdeutsch/how-to-build-a-self-driving-car-in-one-month-d52df48f5b07
+This implementation is based on the one from https://github.com/SullyChen/Autopilot-TensorFlow, and we change the original Dave model to the comma ai's steering model.

Tests/DNN-model/comma-ai-steering-model/run_dataset.py

@@ -22,11 +22,12 @@
 fi = ti.TensorFI(sess, logLevel = 50, name = "convolutional", disableInjections=True)
 
 
-i = 140
+imgIndex = 140
 #while(cv2.waitKey(10) != ord('q')):
 
-for j in range(1):
-    full_image = scipy.misc.imread("driving_dataset/" + str(i) + ".jpg", mode="RGB")
+
+for i in range(1):  # num of imgs to be injected
+    full_image = scipy.misc.imread("driving_dataset/" + str(imgIndex) + ".jpg", mode="RGB")
     image = scipy.misc.imresize(full_image[-150:], [66, 200]) / 255.0
 
     '''    
@@ -52,13 +53,14 @@
 
 
 
-    fi.turnOnInjections()
-    ti.faultTypes.sequentialFIinit()
+    fi.turnOnInjections() 
 
     totalFI = 0.
     sdcCount = 0 
-#    for k in range(3100):
-    while(ti.faultTypes.isKeepDoingFI):
+
+    fiCount = 100
+    for k in range(fiCount): 
+        # steering angle under fault
         degrees = model.y.eval(feed_dict={model.x: [image], model.keep_prob: 1.0})[0][0] * 180.0 / scipy.pi 
 
         totalFI += 1
@@ -88,6 +90,5 @@
 #        dst = cv2.warpAffine(img,M,(cols,rows))
 #        cv2.imshow("steering wheel", dst)
 
-    i += 1 
 
 cv2.destroyAllWindows()