diff --git a/code/nnv/engine/nn/layers/ReluLayer.m b/code/nnv/engine/nn/layers/ReluLayer.m index f7f060102f..380f2ca7fb 100644 --- a/code/nnv/engine/nn/layers/ReluLayer.m +++ b/code/nnv/engine/nn/layers/ReluLayer.m @@ -93,7 +93,7 @@ dp = in_image.depth; c = in_image.numChannel; % transform to star and compute relu reachability - Y = PosLin.reach(in_image.toStar, method, [], relaxFactor); % reachable set computation with ReLU + Y = PosLin.reach(in_image.toStar, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU n = length(Y); % transform back to VolumeStar images(n) = VolumeStar; @@ -105,7 +105,7 @@ w = in_image.width; c = in_image.numChannel; - Y = PosLin.reach(in_image.toStar, method, [], relaxFactor); % reachable set computation with ReLU + Y = PosLin.reach(in_image.toStar, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU n = length(Y); images(n) = ImageStar; % transform back to ImageStar @@ -113,7 +113,7 @@ images(i) = Y(i).toImageStar(h,w,c); end else % star - images = PosLin.reach(in_image, method, [], relaxFactor); % reachable set computation with ReLU + images = PosLin.reach(in_image, method, [], relaxFactor, dis_opt, lp_solver); % reachable set computation with ReLU end end diff --git a/code/nnv/engine/nncs/NonLinearODE.m b/code/nnv/engine/nncs/NonLinearODE.m index d150481079..d467be60b6 100644 --- a/code/nnv/engine/nncs/NonLinearODE.m +++ b/code/nnv/engine/nncs/NonLinearODE.m @@ -313,6 +313,10 @@ function set_output_mat(obj, output_mat) I = init_set.getZono; U = input_set.getZono; + if isempty(U) + U = Star(zeros(input_set.dim,1), zeros(input_set.dim,1)); + U = U.getZono; + end if ~isempty(varargin) if string(varargin{1}) == "poly" || string(varargin{1}) == "lin" || string(varargin{1}) == "lin-adaptive" || string(varargin{1}) == "poly-adaptive" diff --git a/code/nnv/examples/Submission/VNN_COMP2024/config.yaml b/code/nnv/examples/Submission/VNN_COMP2024/config.yaml index 1ed3d08d9e..d55b116f39 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/config.yaml +++ b/code/nnv/examples/Submission/VNN_COMP2024/config.yaml @@ -1,6 +1,6 @@ name: nnv ami: ami-080fce32dc8c15ced -scripts_dir: code/nnv/examples/Submission/VNN_COMP2023/ +scripts_dir: code/nnv/examples/Submission/VNN_COMP2024/ manual_installation_step: True run_installation_script_as_root: False run_post_installation_script_as_root: False diff --git a/code/nnv/examples/Submission/VNN_COMP2024/install_tool.sh b/code/nnv/examples/Submission/VNN_COMP2024/install_tool.sh index a8e0be54d9..e7e16f268f 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/install_tool.sh +++ b/code/nnv/examples/Submission/VNN_COMP2024/install_tool.sh @@ -13,24 +13,27 @@ fi echo "Installing $TOOL_NAME dependencies" -# Install support packages (ONNX importer) -curl --retry 100 --retry-connrefused -L -O https://www.dropbox.com/s/4p17xm4tlm8r9gs/sppFile.zip # need to do this step for R2024a -sleep 60 -sudo unzip sppFile.zip -d /home/ubuntu/toolkit/code/nnv -sudo rm sppFile.zip +ip link show # get mac address (for licensing) -# This is called networks2023, but it contains the MATLAB networks for most of 2022 and 2023 (Do we need this step for 2024?) -cd /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2023/ -curl --retry 100 --retry-connrefused -L -O https://www.dropbox.com/scl/fi/chb0fotern4r5sycz57r9/networks2023.zip?rlkey=1cwvh73b5zgtnb67xdrvlg8am&dl=0 +echo $USER # get usernme (for licensing) -sleep 60 +mkdir ~/.matlab/R2024a_licenses -unzip *.zip* +# INSTALL MPM TO INSTALL ADDITIONAL MATLAB PACKAGES -ip link show # get mac address (for licensing) +apt install wget -echo $USER # get usernme (for licensing) +wget https://www.mathworks.com/mpm/glnxa64/mpm -mkdir ~/.matlab/R2024a_licenses +chmod +x mpm + +./mpm install --release=R2024a --products Deep_Learning_Toolbox_Converter_for_ONNX_Model_Format + +# ADD STEPS TO INSTALL GUROBI + +cd ~/ + +wget https://packages.gurobi.com/11.0/gurobi11.0.2_linux64.tar.gz + +tar xvfz gurobi11.0.2_linux64.tar.gz -# ADD STEPS TO INSTALL GUROBI \ No newline at end of file diff --git a/code/nnv/examples/Submission/VNN_COMP2024/post_install.sh b/code/nnv/examples/Submission/VNN_COMP2024/post_install.sh index f6aa911271..fbaa95c7e0 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/post_install.sh +++ b/code/nnv/examples/Submission/VNN_COMP2024/post_install.sh @@ -10,22 +10,33 @@ ls -al cp -f license.lic /usr/local/matlab/licenses/ -rm /usr/local/matlab/licenses/license_info.xml - -matlab -nodisplay -r "cd /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2023/; prepare_run; quit" +# rm *.zip* -sudo apt install -y python3-pip -pip install numpy +rm /usr/local/matlab/licenses/license_info.xml cd /usr/local/matlab/extern/engines/python python3 -m pip install . # ADD STEPS TO INSTALL GUROBI +cp -f gurobi.lic ~/gurobi1102/ + +echo 'export GUROBI_HOME="~/gurobi1102/linux64"' >> ~/.bashrc +echo 'export GRB_LICENSE_FILE="~/gurobi1102/gurobi.lic"' >> ~/.bashrc +echo 'export PATH="${PATH}:${GUROBI_HOME}/bin"' >> ~/.bashrc +echo 'export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${GUROBI_HOME}/lib"' >> ~/.bashrc + +# Ensure installation is correct +matlab -nodisplay -r "cd /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2024/; prepare_run; quit" + +sudo apt install -y python3-pip +pip install numpy + # TEST IF MATLAB ENGINE IS INSTALLED PROPERLY # START_ENGINE ='import matlab.engine\nimport time\neng = matlab.engine.start_matlab() \nprint(eng) \neng.prepare_run() \nexit()' # python3 -c "exec('import matlab.engine\nimport time\neng = matlab.engine.start_matlab() \nprint(eng) \neng.prepare_run(nargout = 0, background=True) \nexit()')" # TEST IF WE CAN FIND MATLAB -# cd /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2023/ +# cd /home/ubuntu/toolkit/code/nnv/examples/Submission/VNN_COMP2024/ # ./prepare_instance.sh v1 acasxu acas xu + diff --git a/code/nnv/examples/Submission/VNN_COMP2024/prepare_run.m b/code/nnv/examples/Submission/VNN_COMP2024/prepare_run.m index a0b21a7b31..17d5437a95 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/prepare_run.m +++ b/code/nnv/examples/Submission/VNN_COMP2024/prepare_run.m @@ -5,6 +5,10 @@ function prepare_run() % remove paths from any prior installation (if any) % rmpath(genpath('/home/ubuntu/toolkit/code/nnv/')); savepath; +% install gurobi +cd ~/gurobi1102/linux64/matlab; +gurobi_setup; + % installing nnv cd /home/ubuntu/toolkit/code/nnv/; install; @@ -23,8 +27,8 @@ function prepare_run() %matlabshared.supportpkg.setSupportPackageRoot('/usr/local/MATLAB/R2022b'); %addpath(genpath('/usr/local/MATLAB')); -disp("Support package path"); -disp(matlabshared.supportpkg.getSupportPackageRoot); +#disp("Support package path"); +#disp(matlabshared.supportpkg.getSupportPackageRoot); savepath; % quit; diff --git a/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m b/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m index 9b707e2551..1a79227a3c 100644 --- a/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m +++ b/code/nnv/examples/Submission/VNN_COMP2024/run_vnncomp2024_instance.m @@ -91,6 +91,7 @@ % end reachOptions = struct; +reachOptions.lp_solver = "gurobi"; reachOptions.reachMethod = 'approx-star'; % reachOptions.reachMethod = 'exact-star'; % reachOptions.device = 'cpu'; diff --git a/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m b/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m new file mode 100644 index 0000000000..8b97e67dde --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/functions/add_voxels.m @@ -0,0 +1,45 @@ +function I = add_voxels(vol, voxels, noise_disturbance) + % noise_disturnamce can be kept fixed here, more interesting on number + % of voxels changed + + % Return a VolumeStar of a brightening attack on a few pixels + + % Initialize vars + ct = 0; % keep track of pixels modified + flag = 0; % determine when to stop modifying pixels + vol = single(vol); + at_vol = vol; + + % Create brightening attack + for i=1:size(vol,1) + for j=1:size(vol,2) + for k=1:size(vol,3) + if vol(i,j,k) < threshold + at_vol(i,j,k) = 255; + ct = ct + 1; + if ct >= max_pixels + flag = 1; + break; + end + end + end + if flag == 1 + break + end + end + if flag == 1 + break; + end + end + + % Define input set as VolumeStar + dif_vol = -vol + at_vol; + noise = dif_vol; + V(:,:,:,:,1) = vol; % center of set + V(:,:,:,:,2) = noise; % basis vectors + C = [1; -1]; % constraints + d = [1; -1]; % constraints + I = VolumeStar(V, C, d, 1-noise_disturbance, 1); % input set + + +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/functions/remove_voxels.m b/code/nnv/examples/Submission/WiP_3d/functions/remove_voxels.m new file mode 100644 index 0000000000..e69de29bb2 diff --git a/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m b/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m new file mode 100644 index 0000000000..0b8e0b66e5 --- /dev/null +++ b/code/nnv/examples/Submission/WiP_3d/functions/verify_instance_shape.m @@ -0,0 +1,65 @@ +function results = verify_instance_shape(net, vol, target, attack, reachOptions) + % verify medmnist with inputs (input images), targets (labels) and attack + % (struct with adversarial attack info) + % results = verify_medmnist(net, matlabNet, inputs, targets, attack, max_value*, min_value*) + + % Check what type of attack to consider + if strcmp(attack.Name, 'add') || strcmp(attack.Name, 'remove') + max_pixels = attack.max_pixels; + threshold = attack.threshold; + noise_disturbance = attack.noise_de; + else + error("Adversarial attack not supported."); + end + + % Choose attack + if strcmp(attack.Name, 'add') + I = add_voxels(vol, max_pixels, threshold, noise_disturbance); + elseif strcmp(attack.Name, 'remove') + I = remove_voxels(vol, max_pixels, threshold, noise_disturbance); + end + + % Begin analysis + + t = tic; % start timer + + results = zeros(1,2); + + % Check for missclassification + vol = single(vol); + y = net.evaluate(vol); + [~, y] = max(y); + if y ~= target + results(1) = -1; % missclassified + results(2) = toc(t); + return + end + + % Check for falsification + n_samples = 100; % number of random samples to try for falsification + xRand = I.sample(n_samples); + for k = 1:n_samples + x = xRand{k}; + y = net.evaluate(x); + [~,idx] = max(y); + if idx ~= target + results(1) = 0; + results(2) = toc(t); + return + end + end + + % Compute reachability for verification + try + results(1) = net.verify_robustness(I, reachOptions, target); + catch ME + results(1) = -2; + warning(ME.message); + end + + % Save results + results(2) = toc(t); + +end + + diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png new file mode 100644 index 0000000000..df077e7e32 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_100.png new file mode 100644 index 0000000000..0f6c9bdacf Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_1000.png new file mode 100644 index 0000000000..0b793b9ead Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png new file mode 100644 index 0000000000..b386d43d6b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_4.png new file mode 100644 index 0000000000..c03e3540b6 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_50.png new file mode 100644 index 0000000000..89d618c711 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_500.png new file mode 100644 index 0000000000..22b23ccb81 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png new file mode 100644 index 0000000000..4cceb477d4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_100.png new file mode 100644 index 0000000000..f526c90337 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_1000.png new file mode 100644 index 0000000000..42bf7e88a6 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png new file mode 100644 index 0000000000..d18a31a881 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_4.png new file mode 100644 index 0000000000..6eddcba873 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_50.png new file mode 100644 index 0000000000..9d62612208 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_500.png new file mode 100644 index 0000000000..61ee207fbd Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/avgTime_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png new file mode 100644 index 0000000000..499501b223 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_100.png new file mode 100644 index 0000000000..b6dcc81883 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_1000.png new file mode 100644 index 0000000000..a5a17369e1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png new file mode 100644 index 0000000000..ca9cc01148 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_4.png new file mode 100644 index 0000000000..c4af97349f Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_50.png new file mode 100644 index 0000000000..1af3bc1b57 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_500.png new file mode 100644 index 0000000000..d467bc2f5b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png new file mode 100644 index 0000000000..acf2f1c706 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_100.png new file mode 100644 index 0000000000..a915449c7b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_1000.png new file mode 100644 index 0000000000..735af31db4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png new file mode 100644 index 0000000000..739b3b6714 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_4.png new file mode 100644 index 0000000000..851548f758 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_50.png new file mode 100644 index 0000000000..4541a6970d Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_500.png new file mode 100644 index 0000000000..0a42e6d828 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/fracture/verification_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png new file mode 100644 index 0000000000..271105e06f Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_100.png new file mode 100644 index 0000000000..65a0183cb4 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_1000.png new file mode 100644 index 0000000000..8477355511 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png new file mode 100644 index 0000000000..2e72d1145c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_4.png new file mode 100644 index 0000000000..322b2caf7e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_50.png new file mode 100644 index 0000000000..9c7011e489 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_500.png new file mode 100644 index 0000000000..19503b28dc Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png new file mode 100644 index 0000000000..55602353ae Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_100.png new file mode 100644 index 0000000000..c62bbf30e0 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_1000.png new file mode 100644 index 0000000000..bc65efe59d Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png new file mode 100644 index 0000000000..c7fab4ac1b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_4.png new file mode 100644 index 0000000000..7ce085659c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_50.png new file mode 100644 index 0000000000..8999644d63 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_500.png new file mode 100644 index 0000000000..0d802d2f11 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/avgTime_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png new file mode 100644 index 0000000000..ac4de7d928 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_100.png new file mode 100644 index 0000000000..293b67c821 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_1000.png new file mode 100644 index 0000000000..4274507c24 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png new file mode 100644 index 0000000000..09f3430d5e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_4.png new file mode 100644 index 0000000000..ba5b24358b Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_50.png new file mode 100644 index 0000000000..74a5f82425 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_500.png new file mode 100644 index 0000000000..2214c42dc3 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png new file mode 100644 index 0000000000..035baf86fb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_100.png new file mode 100644 index 0000000000..a812a42b50 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_1000.png new file mode 100644 index 0000000000..403c0d397d Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png new file mode 100644 index 0000000000..a5e0e06487 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_4.png new file mode 100644 index 0000000000..35613f269c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_50.png new file mode 100644 index 0000000000..22b30b6904 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_500.png new file mode 100644 index 0000000000..b0378e9bfc Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/nodule/verification_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png new file mode 100644 index 0000000000..568953bcc1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_100.png new file mode 100644 index 0000000000..746dfc9a2a Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_1000.png new file mode 100644 index 0000000000..db15d8ca96 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png new file mode 100644 index 0000000000..af9c970c79 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_4.png new file mode 100644 index 0000000000..8bc0d31b84 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_50.png new file mode 100644 index 0000000000..194246c3e3 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_500.png new file mode 100644 index 0000000000..cef56d2516 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png new file mode 100644 index 0000000000..d130628be2 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_100.png new file mode 100644 index 0000000000..6fdc524d04 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_1000.png new file mode 100644 index 0000000000..6d65c70912 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png new file mode 100644 index 0000000000..0da057ca82 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_4.png new file mode 100644 index 0000000000..655ccbe174 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_50.png new file mode 100644 index 0000000000..4a3cca7732 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_500.png new file mode 100644 index 0000000000..d0fb7e22fc Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/avgTime_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png new file mode 100644 index 0000000000..aff9e11133 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_100.png new file mode 100644 index 0000000000..af8bf367a2 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_1000.png new file mode 100644 index 0000000000..186064d6d3 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png new file mode 100644 index 0000000000..fbc46472fb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_4.png new file mode 100644 index 0000000000..ee5ff70802 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_50.png new file mode 100644 index 0000000000..5430f60b6c Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_500.png new file mode 100644 index 0000000000..e7757b5fbb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png new file mode 100644 index 0000000000..68d0ce9dbf Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_100.png new file mode 100644 index 0000000000..7e2e838879 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_1000.png new file mode 100644 index 0000000000..9253d08f4a Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png new file mode 100644 index 0000000000..cba4a42beb Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_4.png new file mode 100644 index 0000000000..34e1c311d8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_50.png new file mode 100644 index 0000000000..a05f027ac8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_500.png new file mode 100644 index 0000000000..ab290f4712 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/organ/verification_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png new file mode 100644 index 0000000000..c6143b5033 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_100.png new file mode 100644 index 0000000000..76e4230df8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_1000.png new file mode 100644 index 0000000000..2c061bf6e1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png new file mode 100644 index 0000000000..c1b0488b5e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_4.png new file mode 100644 index 0000000000..0fb6109b40 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_50.png new file mode 100644 index 0000000000..183292c5b9 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_500.png new file mode 100644 index 0000000000..bcc410bc4a Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png new file mode 100644 index 0000000000..4e81591e08 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_100.png new file mode 100644 index 0000000000..eb9ecf7749 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_1000.png new file mode 100644 index 0000000000..9f6a343798 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png new file mode 100644 index 0000000000..eef06f7cf8 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_4.png new file mode 100644 index 0000000000..bdc7d654e7 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_50.png new file mode 100644 index 0000000000..6ba03bbe56 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_500.png new file mode 100644 index 0000000000..d1804d018a Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/avgTime_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png new file mode 100644 index 0000000000..d34e16fa25 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_100.png new file mode 100644 index 0000000000..b0c7849f33 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_1000.png new file mode 100644 index 0000000000..047c5243ad Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png new file mode 100644 index 0000000000..83c5d26374 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_4.png new file mode 100644 index 0000000000..a86d90b0a9 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_50.png new file mode 100644 index 0000000000..4e2e92ccd1 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_500.png new file mode 100644 index 0000000000..d6cf121774 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_bright_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png new file mode 100644 index 0000000000..eb979c7240 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_10.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_100.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_100.png new file mode 100644 index 0000000000..a8c3bc062e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_100.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_1000.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_1000.png new file mode 100644 index 0000000000..92132bce63 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_1000.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png new file mode 100644 index 0000000000..d08a7be066 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_2.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_4.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_4.png new file mode 100644 index 0000000000..896af079be Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_4.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_50.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_50.png new file mode 100644 index 0000000000..81fc7e7c49 Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_50.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_500.png b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_500.png new file mode 100644 index 0000000000..a8c3bc062e Binary files /dev/null and b/code/nnv/examples/Submission/WiP_3d/plots/synapse/verification_dark_500.png differ diff --git a/code/nnv/examples/Submission/WiP_3d/plots/verification_fracture_bright_2.png b/code/nnv/examples/Submission/WiP_3d/plots/verification_fracture_bright_2.png deleted file mode 100644 index 14759f8deb..0000000000 Binary files a/code/nnv/examples/Submission/WiP_3d/plots/verification_fracture_bright_2.png and /dev/null differ diff --git a/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m b/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m index 38b8fb4d00..f5061dd62c 100644 --- a/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m +++ b/code/nnv/examples/Submission/WiP_3d/verify_adrenal.m @@ -31,139 +31,35 @@ reachOptions.lp_solver = "gurobi"; -%% Attack 1 - -% adv_attack = struct; -%??????? - -% results = zeros(N,2); % verification result, time - -% verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end - -% save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 2 - -adv_attack = struct; - -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 3 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 4 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 5 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 6 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 7 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - - -%% Attack 8 - -% adv_attack = struct; -% -% results = zeros(N,2); % verification result, time -% -% % verify volumes -% parfor i=1:N -% img = squeeze(inputs(:,:,:,:,i)); -% target = targets(i); -% results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); -% end -% -% % save results -% save("results/verification_adrenal_"+adv_attack.Name+"_" +adv_attack.noise_de +"_" +adv_attack.max_pixels + ".mat", "results"); - +%% Verification analysis +for a=advType + for mp=maxpixels + for ep=epsilon + + % 1) Initialize results var + results = zeros(N,2); + + % 2) Create adversarial attack + adv_attack = struct; + adv_attack.Name = a; % bright or dark + if strcmp(a, "bright") + adv_attack.threshold = threshold(1); % perturb pixels below this value + else + adv_attack.threshold = threshold(2); % perturb pixels below this value + end + adv_attack.max_pixels = mp; % Max number of pixels to modify from input image + adv_attack.noise_de = ep/255; % disturbance (noise) on pixels + + % 3) Begin verification analysis + for i=1:N + img = squeeze(inputs(:,:,:,:,i)); + target = targets(i); + results(i,:) = verify_instance_3d(net, img, target, adv_attack, reachOptions); + end + + % 4) % save results + save("results/verification_adrenal_"+ a +"_" + ep +"_" + mp + ".mat", "results"); + + end + end +end \ No newline at end of file diff --git a/code/nnv/examples/Submission/WiP_3d/visualize_results.m b/code/nnv/examples/Submission/WiP_3d/visualize_results.m index f1afc35b31..4b273482d1 100644 --- a/code/nnv/examples/Submission/WiP_3d/visualize_results.m +++ b/code/nnv/examples/Submission/WiP_3d/visualize_results.m @@ -9,6 +9,12 @@ maxpixels = ["50", "100", "500", "1000"]; epsilon = ["2", "4", "10"]; % epsilon/255 +% Initialize directories +mkdir('plots'); +for i=datasets + mkdir("plots/"+i); +end + %% Visualize verification results/trends % What do we want to show? @@ -25,16 +31,19 @@ % - Do we need more samples? -% Time for plots +% Time for plots (maxpixels variable) for ds = datasets for adv = attackType for ep = epsilon + % Initialize vars to plot sat = []; unsat = []; unk = []; miss = []; - avgTime = []; + avgVT = []; + avgRT = []; + % Get data for mp = maxpixels resFile = "verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; @@ -43,16 +52,110 @@ unsat = [unsat, res.unsat]; unk = [unk, res.unknown]; miss = [miss, res.misclassified]; - avgTime = [avgTime, res.avgTime]; + avgVT = [avgVT, res.avgTime]; % average computation time to verify one instance + avgRT = [avgRT, res.avgRT]; % average computation time to compute the reachable sets (miss and sat do not count here) end counts = [unsat; sat; unk; miss]; + + + % Create figure (results) + f = figure('visible','off'); + bar(1:4, counts','stacked') % plot verification results + grid; + % set values for x-axis + xticks([1 2 3 4]); + xticklabels(maxpixels); + % axis labels + xlabel('Max pixels perturbed'); + ylabel('# instances') + % legend leg = {"unsat", "sat", "unknown", "missclass"}; - % Create figure - f = figure; - bar(1:4, counts','stacked') % change ticks and label later on + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/verification_" + adv + "_" + ep + ".png"); + + % create figure (time) + f = figure('visible','off'); + % plot computation time results + plot(1:4, avgVT, 'r--o'); + hold on; + plot(1:4, avgRT, 'b--v'); + % set values for x-axis + xticks([1 2 3 4]); + xticklabels(maxpixels) + % axis labels + xlabel('Max pixels perturbed'); + ylabel("Average Time (s)") + % legend + leg = {"all", "unsat & unknown"}; + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/avgTime_" + adv + "_" + ep + ".png"); + + end + end +end + +% Time for plots (epsilon variable) +for ds = datasets + for adv = attackType + for mp = maxpixels + + % Initialize vars to plot + sat = []; + unsat = []; + unk = []; + miss = []; + avgVT = []; + avgRT = []; + + % Get data + for ep = epsilon + resFile = "verification_" + ds + "_" + adv + "_" + ep + "_" + mp +".mat"; + res = summarize_results(resFile); + sat = [sat, res.sat]; + unsat = [unsat, res.unsat]; + unk = [unk, res.unknown]; + miss = [miss, res.misclassified]; + avgVT = [avgVT, res.avgTime]; % average computation time to verify one instance + avgRT = [avgRT, res.avgRT]; % average computation time to compute the reachable sets (miss and sat do not count here) + end + counts = [unsat; sat; unk; miss]; + + % Create figure (results) + f = figure('visible','off'); + bar(1:3, counts','stacked') % plot results grid + % set values for x-axis + xticks([1 2 3]); + xticklabels(epsilon) + % axis labels + xlabel('epsilon (\epsilon)'); + ylabel('# instances') + % legend + leg = {"unsat", "sat", "unknown", "missclass"}; legend(leg, 'Location', 'best'); - saveas(f, "plots/verification_" + ds + "_" + adv + "_" + ep+".png"); + % save figure + saveas(f, "plots/" + ds + "/verification_" + adv + "_" + mp + ".png"); + + % create figure (time) + f = figure('visible','off'); + % plot timing results + plot(1:3, avgVT, 'r--o'); + hold on; + plot(1:3, avgRT, 'b--v'); + % set values for x-axis + xticks([1 2 3]); + xticklabels(epsilon) + % axis labels + xlabel('epsilon (\epsilon)'); + ylabel("Average Time (s)") + % legend + leg = {"all", "unsat & unknown"}; + legend(leg, 'Location', 'best'); + % save figure + saveas(f, "plots/" + ds + "/avgTime_" + adv + "_" + mp + ".png"); + end end end @@ -61,20 +164,34 @@ %% Helper functions function summary = summarize_results(resFile) + % Provide details from results file summary = struct; data = load(resFile); results = data.results; + % Total number of samples examined summary.N = length(results); + % Verified (unsat) summary.unsat = sum(results(:,1) == 1); + % Falsified (sat) from input set summary.sat = sum(results(:,1) == 0); + % Unknown (not using exact) summary.unknown = sum(results(:,1) == 2); + % Misclasified (sat) of original image summary.misclassified = sum(results(:,1) == -1); + % Also possible -2 (error), which may be out of memory (most common here) summary.avgTime = sum(results(:,2))/summary.N; + + % Find instances where reachability is needed + x = [find(results(:,1)==2); find(results(:,1)==1)]; + nx = length(x); % how many instances reachability is computed for + + % Average time only for unknown and unsat properties (reachsets computed) + summary.avgRT = sum(results(x,2))/nx; end \ No newline at end of file