diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..d532814 --- /dev/null +++ b/.clang-format @@ -0,0 +1,196 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +# Start Customization +# +AlignArrayOfStructures: Right +AlignConsecutiveMacros: AcrossEmptyLinesAndComments +AlignConsecutiveAssignments: AcrossEmptyLinesAndComments +AlignConsecutiveBitFields: Consecutive +AllowShortBlocksOnASingleLine: Always +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLoopsOnASingleLine: true +BreakBeforeBraces: Custom +ColumnLimit: 120 +IndentWidth: 4 +NamespaceIndentation: All +PointerAlignment: Left +ReferenceAlignment: Pointer +QualifierAlignment: Left +SeparateDefinitionBlocks: Always +SortIncludes: CaseInsensitive +Standard: c++17 +# End Customization +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortLambdasOnASingleLine: All +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +AttributeMacros: + - __capability +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: true + AfterControlStatement: true + AfterEnum: false + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: true + BeforeCatch: false + BeforeElse: true + BeforeLambdaBody: false + BeforeWhile: true + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: false + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: true +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +PackConstructorInitializers: BinPack +BasedOnStyle: '' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +AllowAllConstructorInitializersOnNextLine: true +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Merge +IncludeCategories: + - Regex: '<[[:alnum:]_]+>' + Priority: -1 + CaseSensitive: false + - Regex: '<[[:alnum:]._/]+>' + Priority: 1 + CaseSensitive: false + - Regex: '<[[:alnum:]._]+>' + Priority: 2 + CaseSensitive: false + - Regex: '\/' + Priority: 3 + CaseSensitive: false + - Regex: '.*' + Priority: 4 + CaseSensitive: false +IncludeIsMainRegex: '"$"' +IncludeIsMainSourceRegex: '"$"' +IndentAccessModifiers: false +IndentCaseLabels: false +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentRequires: false +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +LambdaBodyIndentation: OuterScope +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PenaltyIndentedWhitespace: 0 +PPIndentWidth: -1 +ReflowComments: true +RemoveBracesLLVM: false +ShortNamespaceLines: 1 +SortJavaStaticImport: Before +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + BeforeNonEmptyParentheses: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +BitFieldColonSpacing: Both +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseCRLF: false +UseTab: Never +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE + - NS_SWIFT_NAME + - CF_SWIFT_NAME +... + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f15c40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +build/ +data/MNIST/ +*.eps +*.hd5 +*.obj diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a52d0cc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "externals/ma_libs"] + path = externals/ma_libs + url = https://github.com/azimonti/ma_libs.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..dc6c23e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,106 @@ +cmake_minimum_required(VERSION 3.13.4) +project(nnetworks_example) +set (PROJECT_VERSION "1.0" ) +project(${PROJECT_NAME} VERSION ${PROJECT_VERSION}) + +# location of the ma_libs +set(MA_LIBS_ROOT "${CMAKE_SOURCE_DIR}/externals/ma_libs") +set(MA_LIBS_CPP_ROOT "${MA_LIBS_ROOT}/cpp") + + +list(APPEND CMAKE_MODULE_PATH "${MA_LIBS_ROOT}/cmake_modules") +include( COMPILERCOMMON ) +include( COMPILERCPP ) +include( FindLIBS ) +set(CMAKE_CXX_STANDARD 17 ) +add_definitions( -DLOGGING -DCOUTEXT ) + +if(MSVC) + generic_libs_find(hdf5 ON ) + generic_libs_find(zlib ON ) + include_directories( ${HDF5_INCLUDE_DIRS}/src ) + include_directories( ${HDF5_INCLUDE_DIRS2} ) + link_directories( ${ZLIB_LIBRARY_PATH} ) +else() + generic_libs_find(hdf5 OFF ) + add_compile_options(-Wall -Wextra -pedantic -Wconversion -Wno-float-conversion) +endif() + +include_directories( ${MATPLOT_INCLUDE_DIRS} ) +include_directories( ${HDF5_INCLUDE_DIRS} ) +include_directories( ${MA_LIBS_CPP_ROOT} ) +include_directories( ${MA_LIBS_CPP_ROOT}/math ) +include_directories( ${MA_LIBS_CPP_ROOT}/libnn/src ) +include_directories( ${MA_LIBS_CPP_ROOT}/utils ) + +add_definitions( -DLOGGING ) + +set ( SRCS + ./src/mnist.cpp + ${MA_LIBS_CPP_ROOT}/utils/log/log.cpp + ) + +add_library( objnnetworks OBJECT ${SRCS} ) +set_property( TARGET objnnetworks PROPERTY POSITION_INDEPENDENT_CODE 1 ) +if(UNITYBUILD) + set_property( TARGET objnnetworks PROPERTY UNITY_BUILD ON ) +endif() + +project(network1_bin) +set ( SRCS ./src/main1.cpp ) + +link_directories( ${LIBS_DIR} ) +link_directories( ${HDF5_LIBRARY_PATH} ) +add_executable( ${PROJECT_NAME} ${SRCS} $ ) +if(UNITYBUILD) + set_property( TARGET ${PROJECT_NAME} PROPERTY UNITY_BUILD ON ) +endif() +if(MSVC) + target_link_libraries( ${PROJECT_NAME} + debug libhdf5_D zlibstaticd + optimized libhdf5 zlibstatic ) +else() + target_link_libraries( ${PROJECT_NAME} + debug hdf5 + optimized hdf5 ) +endif() + +link_directories( ${MA_LIBS_ROOT}/build/${CMAKE_BUILD_TYPE} ) + +project(network2_bin) +set ( SRCS ./src/main2.cpp ) + +link_directories( ${LIBS_DIR} ) +link_directories( ${HDF5_LIBRARY_PATH} ) +add_executable( ${PROJECT_NAME} ${SRCS} $ ) +if(UNITYBUILD) + set_property( TARGET ${PROJECT_NAME} PROPERTY UNITY_BUILD ON ) +endif() +if(MSVC) + target_link_libraries( ${PROJECT_NAME} + debug libhdf5_D debug zlibstaticd debug nnd + optimized libhdf5 optimized zlibstatic nn ) +else() + target_link_libraries( ${PROJECT_NAME} + debug hdf5 debug nnd + optimized hdf5 optimized nn) +endif() + +project(network3_bin) +set ( SRCS ./src/main3.cpp ) + +link_directories( ${LIBS_DIR} ) +link_directories( ${HDF5_LIBRARY_PATH} ) +add_executable( ${PROJECT_NAME} ${SRCS} $ ) +if(UNITYBUILD) + set_property( TARGET ${PROJECT_NAME} PROPERTY UNITY_BUILD ON ) +endif() +if(MSVC) + target_link_libraries( ${PROJECT_NAME} + debug libhdf5_D debug zlibstaticd debug nnd + optimized libhdf5 optimized zlibstatic optimized nn) +else() + target_link_libraries( ${PROJECT_NAME} + debug hdf5 debug nnd + optimized hdf5 optimized nn) +endif() diff --git a/README.md b/README.md index 77925be..c03b571 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,128 @@ # MNIST-classification + Repository demonstrating neural network training for MNIST digit classification. + +# Getting Started + +To get started with the neural networks: + +1. Clone the repository: + ``` + git clone https://github.com/azimonti/MNIST-classification.git + ``` +2. Navigate to the repository directory: + ``` + cd MNIST-classification + ``` +3. Initialize and update the submodules: + ``` + git submodule update --init --recursive + ``` + +3. Compile the libraries in `ma_libs` + ``` + cd external/ma_libs + ./cbuild.sh -t Debug + ./cbuild.sh -t Release + cd .. + ``` + + If any error or missing dependencies please look at the instructions [here](https://github.com/azimonti/ma_libs) + +4. Dowload the MNIST data (source links were retrieved from the page [here](https://github.com/cvdfoundation/mnist) as the original page gives a 403 Forbidden error), but as far as are the original files and are put in the `data/MNIST/` directory it will be fine. + + ``` + bash download_mnist_data.sh + ``` + +5. Compile the binaries + ``` + ./cbuild.sh -t Release (or -t Debug) + ``` + +6. Create the configuation files + ``` + python nn_config.py + ``` + +7. Run the simulations + ``` + ./build/Release/network1_bin + ./build/Release/network2_bin --training_start + ./build/Release/network3_bin --training_start + ``` + +## Program 1 + +The program `network1_bin` is designed to load and display sample MNIST images and their corresponding labels. It uses a simple logging system to either log to the console or to a file based on the `bFileLog` flag. + +The program prints a specific image to the console and outputs the corresponding label, providing a visual and numeric understanding of the dataset. + +``` +Image and label num is: 10000 +Image rows: 28, cols: 28 +Sample : 36 - Results is: 7 +0000000000000000000000000000 +0000000000000000000000000000 +0000000000000000000000000000 +0000000000000000000000000000 +0000000000000000000000000000 +0000000000000000000000000000 +0000000000000000000000000000 +0000001111111111111100000000 +0000011111111111111110000000 +0000011111110000011110000000 +0001111111100000001110000000 +0001111100000000011110000000 +0001110000000000011100000000 +0000000000000000111100000000 +0000000000000000111000000000 +0000000000000001111000000000 +0000000000000001111000000000 +0000000000000011100000000000 +0000000000000111111110000000 +0000000000001111111110000000 +0000000000001111111110000000 +0000000000001111100000000000 +0000000000000111000000000000 +0000000000000111000000000000 +0000000000000111000000000000 +0000000000000111000000000000 +0000000000000111000000000000 +0000000000000000000000000000 +0000000100 +``` + +## Program 2 + +The program `network2_bin` is designed to handle both training and testing of a neural network model using the Stochastic Gradient Descent (SGD) algorithm on the MNIST dataset. It supports the following modes based on command-line arguments: + +Command-line Argument Handling: +- `--training_start`: Starts training from scratch. +- `--training_continue`: Continues training from a previously saved state. +- `--testing`: Skips training and directly proceeds with testing the model. + +Training and Testing: + +- The network trains using the SGD algorithm, where `nepoch` determines the number of epochs (iterations) for training. +- After training, it can be used for testing on the MNIST dataset. + +Performance: The program achieves a good accuracy with 5 epochs, showing more tha 90% accuracy. + +## Program 3 + +The program `network3_bin` uses a Genetic Algorithm (GA) for training a neural network on the MNIST dataset. Like the other programs, it accepts command-line arguments to control whether it should start training from scratch, continue training, or run in testing mode: + +Command-line Argument Handling: +- `--training_start`: Starts training from scratch. +- `--training_continue`: Continues training from a previously saved state. +- `--testing`: Skips training and directly proceeds with testing the model. + +Training and Testing: + +- This program replaces the Stochastic Gradient Descent (SGD) method with a Genetic Algorithm for optimizing the neural network's weights. +- GA Characteristics: Evolutionary processes such as selection, crossover, and mutation are applied across generations to find an optimal set of weights for the network. + +Performance: Despite running for 3000 generations, the accuracy remains low, around 30%. This slow convergence is expected because Genetic Algorithms are not well-suited for optimizing neural networks on tasks like MNIST digit classification, where gradient-based methods such as SGD are more effective. + +The program not only highlights the limitations of Genetic Algorithms for efficient convergence on tasks like MNIST training but also serves as a demonstration of how to use my library, which can be applied to other tasks more suited for GA or where SGD is not applicable, such as simulating interactions in a game. diff --git a/cbuild.sh b/cbuild.sh new file mode 100755 index 0000000..6adfd66 --- /dev/null +++ b/cbuild.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +displayusage() { + echo " =================================================================================== " + echo "| Usage: |" + echo "| cbuild.sh OPTS |" + echo "| available options [OPTS]: |" + echo "| -b) --build) automatically updates the build when necessary |" + echo "| -c) --clean) removes build dirs |" + echo "| -d) --dry-run) creates the make file without building |" + echo "| -f) --force) forces an update of the build |" + echo "| -h) --help) print this help |" + echo "| -m) --make) performs make |" + echo "| -n) --nproc) sets the number of parallel processing (default nproc -1) |" + echo "| -o) --build-one) build a single test. Equivalent to \"-w -DBUILDSINGLE=NAME\"|" + echo "| -r) --recompile) continuously build when a file is saved |" + echo "| -s) --build-suite) build test suite. Equivalent to \"-w -DBUILDSUITE=ON\" |" + echo "| -t) --build-type) specifies a different cmake build type (e.g. \"-t Debug\") |" + echo "| -u) --no-unity-build) do not use unity build. Equivalent to \"-w -NOUNITYBUILD=ON\" |" + echo "| -w) --cmake-params) specifies cmake options in quoted (e.g. \"-DVAR=value\") |" + echo "| -z) --analyze) run scan-build |" + echo "| [no arguments] automatically updates the build when necessary |" + echo " =================================================================================== " +} + +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) MACHINE=linux;; + Darwin*) MACHINE=macos;; + CYGWIN*) MACHINE=win;; + MINGW*) MACHINE=win;; + *) echo "unsupported architecture"; exit 1 +esac + +# Set all option to FALSE +ANALYZE="FALSE" +BUILD="FALSE" +CLEANBUILD="FALSE" +DRY_RUN="FALSE" +UPDATEMAKEFILES="FALSE" +EMAKE="FALSE" +CONTINUOUSCOMPILE="FALSE" + +BASHSCRIPTDIR="$(cd "$(dirname "$0")" || exit; pwd)" +CURRENTDIR="$(pwd)" +SOURCEDIR="${CURRENTDIR}" + +if [[ ! -f "${SOURCEDIR}/CMakeLists.txt" ]] ; then + echo "CMakeLists.txt not found. Exiting..." + exit 1 +fi + +update_makefiles(){ + mkdir -p "${BUILDDIR}" + CURDIR="$(pwd)" + cd "${BUILDDIR}" || exit + + if [ "${MACHINE}" == "macos" ]; then + cmake "${SOURCEDIR}" -DCMAKE_BUILD_TYPE="${BUILDTYPE:-Release}" ${CMAKEOPTS:+$CMAKEOPTS} + elif [ "${MACHINE}" == "linux" ]; then + cmake "${SOURCEDIR}" -DCMAKE_BUILD_TYPE="${BUILDTYPE:-Release}" ${CMAKEOPTS:+$CMAKEOPTS} + elif [ "${MACHINE}" == "win" ]; then + WINARCH="x64" + if [[ $PROCESSOR_IDENTIFIER == *"ARM"* ]]; then WINARCH="ARM64"; fi + cmake -A "${WINARCH}" "${SOURCEDIR}" -DCMAKE_BUILD_TYPE="${BUILDTYPE:-Release}" ${CMAKEOPTS:+$CMAKEOPTS} + fi + CMAKE_RET=$? + if [ $CMAKE_RET -ne 0 ] ; then exit ${CMAKE_RET}; fi + local BT="${BUILDTYPE:-Release}" + echo "$(tr '[:lower:]' '[:upper:]' <<< "${BT:0:1}")$(tr '[:upper:]' '[:lower:]' <<< "${BT:1}")" > "${BUILDTYPEFILE}" + cd "${CURDIR}" || exit + MAKEFILEUPDATED="TRUE" +} + +build(){ + if [[ ! -f "${NBFILES}" ]] ; then mkdir -p "${BUILDDIR}"; echo 0 > "${NBFILES}" ; fi + PREVFILES=$(<"${NBFILES}") + CURRFILES=$(find "${SOURCEDIR}" | wc -l) + + if [[ "${MAKEFILEUPDATED}" == "TRUE" ]] || [[ "${CURRFILES}" -ne "${PREVFILES}" ]] ; then update_makefiles ; fi + + if [ "${MACHINE}" == "win" ]; then + CMAKE_BUILD_PARALLEL_LEVEL=3 cmake --build "${BUILDDIR}" --config "${BUILDTYPE:-Release}" + else + cmake --build "${BUILDDIR}" --config "${BUILDTYPE:-Release}" -j "${NPROC}" + fi + CMAKE_RET=$? + echo "${CURRFILES}" > "${NBFILES}" + if [ $CMAKE_RET -ne 0 ] ; then exit ${CMAKE_RET}; fi +} + +analyze(){ + mkdir -p "${BUILDDIR}" + cd "${BUILDDIR}" || exit + + if [ "${MACHINE}" == "macos" ]; then + scan-build cmake "${SOURCEDIR}" -DCMAKE_BUILD_TYPE="${BUILDTYPE:-Release}" ${CMAKEOPTS:+$CMAKEOPTS} + else + echo "unsupported architecture"; exit 1 + fi + CMAKE_RET=$? + if [ $CMAKE_RET -ne 0 ] ; then exit ${CMAKE_RET}; fi + local BT="${BUILDTYPE:-Release}" + echo "$(tr '[:lower:]' '[:upper:]' <<< "${BT:0:1}")$(tr '[:upper:]' '[:lower:]' <<< "${BT:1}")" > "${BUILDTYPEFILE}" + cd - || exit + + scan-build cmake --build "${BUILDDIR}" --config "${BUILDTYPE:-Release}" -j "${NPROC}" + CMAKE_RET=$? + if [ $CMAKE_RET -ne 0 ] ; then exit ${CMAKE_RET}; fi +} + +for arg in "$@"; do + shift + case "$arg" in + "--build") set -- "$@" "-b" ;; + "--clean") set -- "$@" "-c" ;; + "--dry-run") set -- "$@" "-d" ;; + "--force") set -- "$@" "-f" ;; + "--help") set -- "$@" "-h" ;; + "--make") set -- "$@" "-m" ;; + "--nproc") set -- "$@" "-n" ;; + "--build-one") set -- "$@" "-o" ;; + "--recompile") set -- "$@" "-r" ;; + "--build-suite") set -- "$@" "-s" ;; + "--build-type") set -- "$@" "-t" ;; + "--no-unity-build") set -- "$@" "-u" ;; + "--cmake-params") set -- "$@" "-w" ;; + "--analyze") set -- "$@" "-z" ;; + *) set -- "$@" "$arg";; + esac +done + +# Parse short options +OPTIND=1 +while getopts "bcdfhmn:o:rst:uw:z?" opt +do + case "$opt" in + "b") BUILD="TRUE";; + "c") CLEANBUILD="TRUE";; + "d") UPDATEMAKEFILES="TRUE"; DRY_RUN="TRUE" ;; + "f") UPDATEMAKEFILES="TRUE";; + "h") displayusage; exit 0;; + "m") EMAKE="TRUE";; + "n") NPROC="${OPTARG}";; + "o") CMAKEOPTS+=" -DBUILDSINGLE=${OPTARG} "; UPDATEMAKEFILES="TRUE";; + "r") CONTINUOUSCOMPILE="TRUE";; + "s") CMAKEOPTS+=" -DBUILDSUITE=ON "; UPDATEMAKEFILES="TRUE";; + "t") BUILDTYPE=${OPTARG}; UPDATEMAKEFILES="TRUE";; + "u") CMAKEOPTS+=" -DNOUNITYBUILD=ON "; UPDATEMAKEFILES="TRUE";; + "w") CMAKEOPTS+="${OPTARG} "; UPDATEMAKEFILES="TRUE";; + "z") ANALYZE="TRUE" ;; + "?") displayusage; exit 0;; + esac +done + +shift "$((OPTIND-1))" + +CFG=${BUILDTYPE:-Release} +CFG=$(echo "${CFG}" | tr '[:upper:]' '[:lower:]' ) +BUILDDIR="${CURRENTDIR}/build/master/${CFG}" +NBFILES="${BUILDDIR}/.nbfiles" +BUILDTYPEFILE="${BUILDDIR}/.buildtype" + +if [[ -z "${NPROC}" ]]; then (( NPROC = $(nproc) - 1 )); fi + +if [[ "${CLEANBUILD}" == "TRUE" ]] || [[ "${ANALYZE}" == "TRUE" ]] ; then + read -r -p "Are you sure? [y/N] " CLEANCONFIRM + case "${CLEANCONFIRM}" in + [yY][eE][sS]|[yY]) + rm -rf "${ROOTDIR}"/_bin/test "${BUILDDIR}";; + *) + exit 0;; + esac +fi + +if [[ "${BUILD}" == "TRUE" ]] ; then build; exit 0; fi + +if [[ "${EMAKE}" == "TRUE" ]] ; then emake; exit 0; fi + +if [[ "${ANALYZE}" == "TRUE" ]] ; then analyze; exit 0; fi + +if [[ "${UPDATEMAKEFILES}" == "TRUE" ]] ; then update_makefiles; fi + +if [[ "${DRY_RUN}" == "TRUE" ]] ; then exit 0; fi + +if [[ "${CONTINUOUSCOMPILE}" == "TRUE" ]] ; then + find "${SOURCEDIR}" -not \( -path "${BUILDDIR}" -prune \) -type f | entr -s "${BASHSCRIPTDIR}/cbuild.sh" +else + build +fi diff --git a/download_mnist_data.sh b/download_mnist_data.sh new file mode 100755 index 0000000..b10c7b3 --- /dev/null +++ b/download_mnist_data.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Define the MNIST directory and base URL +MNIST_DIR="data/MNIST" +BASE_URL="https://storage.googleapis.com/cvdf-datasets/mnist/" + +# Create the directory if it doesn't exist +mkdir -p "$MNIST_DIR" + +# Array of filenames to check and download +FILES=( + "train-images-idx3-ubyte" + "train-labels-idx1-ubyte" + "t10k-images-idx3-ubyte" + "t10k-labels-idx1-ubyte" +) + +# Loop through the files, check if they exist with or without .gz, and download if necessary +for FILE in "${FILES[@]}"; do + FILE_PATH="$MNIST_DIR/$FILE" + FILE_GZ_PATH="$FILE_PATH.gz" + + # Check if both the gzipped and unzipped versions are missing + if [ ! -f "$FILE_PATH" ] && [ ! -f "$FILE_GZ_PATH" ]; then + echo "Downloading $FILE.gz..." + wget -P "$MNIST_DIR" "${BASE_URL}${FILE}.gz" + else + echo "$FILE or $FILE.gz already exists, skipping download." + fi + + # Unzip if the gzipped version is found + if [ -f "$FILE_GZ_PATH" ]; then + echo "Unzipping $FILE_GZ_PATH..." + gunzip "$FILE_GZ_PATH" + fi +done diff --git a/externals/ma_libs b/externals/ma_libs new file mode 160000 index 0000000..1ae0a6f --- /dev/null +++ b/externals/ma_libs @@ -0,0 +1 @@ +Subproject commit 1ae0a6f92ea5bb51f12f96629ca405e45cf28e3f diff --git a/nn_config.py b/nn_config.py new file mode 100644 index 0000000..636f13b --- /dev/null +++ b/nn_config.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +''' +/************************/ +/* nn.py */ +/* Version 1.0 */ +/* 2023/03/08 */ +/************************/ +''' + +import h5py +import numpy as np +from os import makedirs +import sys + +CONFDIR = "./build/config" +H5FILE2 = "nn2.hd5" +H5FILE3 = "nn3.hd5" + + +def create_folder(): + makedirs(CONFDIR, exist_ok=True) + + +def create_config2(): + with h5py.File(CONFDIR + '/' + H5FILE2, 'w') as f: + NN = 'nn2' + s = np.int8(list(b"network_sgd")) + f.create_dataset(NN + '/nname', data=s) + s = np.int8(list(b"./build/data_network2.hd5")) + f.create_dataset(NN + '/data_file', data=s) + # first set of parameters + CSET1 = 'cfg_1' + DSET1 = NN + '/' + CSET1 + v = np.int64([784, 30, 10]) + f.create_dataset(DSET1 + '/size', data=v) + i = np.int64([5]) + f.create_dataset(DSET1 + '/nEpochs', data=i) + i = np.int64([10]) + f.create_dataset(DSET1 + '/miniBatchSize', data=i) + d = np.double([3.0]) + f.create_dataset(DSET1 + '/eta', data=d) + + CSET2 = 'cfg_2' + v = np.int64([784, 64, 16, 10]) + DSET2 = NN + '/' + CSET2 + f.create_dataset(DSET2 + '/size', data=v) + i = np.int64([5]) + f.create_dataset(DSET2 + '/nEpochs', data=i) + i = np.int64([10]) + f.create_dataset(DSET2 + '/miniBatchSize', data=i) + d = np.double([3.0]) + f.create_dataset(DSET2 + '/eta', data=d) + + # set the current set + s = np.int8(list(bytes(CSET1, 'ascii'))) + f.create_dataset(NN + '/current_set', data=s) + + +def create_config3(): + with h5py.File(CONFDIR + '/' + H5FILE3, 'w') as f: + NN = 'nn3' + s = np.int8(list(b"network_ga")) + f.create_dataset(NN + '/nname', data=s) + s = np.int8(list(b"./build/data_network3.hd5")) + f.create_dataset(NN + '/data_file', data=s) + # first set of parameters + CSET1 = 'cfg_1' + DSET1 = NN + '/' + CSET1 + v = np.int64([784, 30, 10]) + f.create_dataset(DSET1 + '/size', data=v) + i = np.int64([5000]) + f.create_dataset(DSET1 + '/nGenerations', data=i) + i = np.int64([200]) + f.create_dataset(DSET1 + '/BatchSize', data=i) + + # set the current set + s = np.int8(list(bytes(CSET1, 'ascii'))) + f.create_dataset(NN + '/current_set', data=s) + + +def main(): + create_folder() + create_config2() + create_config3() + + +if __name__ == '__main__': + if sys.version_info[0] < 3: + raise 'Must be using Python 3' + main() diff --git a/src/main1.cpp b/src/main1.cpp new file mode 100644 index 0000000..d464075 --- /dev/null +++ b/src/main1.cpp @@ -0,0 +1,34 @@ +/************************/ +/* main1.cpp */ +/* Version 1.0 */ +/* 2023/02/05 */ +/************************/ + +#include +#include +#include +#include +#include "log/log.h" +#include "mnist.h" + +constexpr const char* const IMAGESDIR = "./data/MNIST"; + +int main() +{ + bool bFileLog = false; + LOGGER_PARAM(logging::LEVELMAX, logging::INFO); + LOGGER_PARAM(logging::LOGLINE, true); + LOGGER_PARAM(logging::LOGTIME, true); + if (bFileLog) + { + LOGGER_PARAM(logging::FILENAME, "out1.log"); + LOGGER_PARAM(logging::FILEOUT, true); + } + nn::MNIST img = nn::MNIST(IMAGESDIR, false, true); + size_t lb = 36; + std::cout << "Sample : " << lb << " - Results is: " << img.LabelNumeric(lb) << std::endl; + img.PrintImage(lb); + img.PrintLabel(lb); + + return 0; +} diff --git a/src/main2.cpp b/src/main2.cpp new file mode 100644 index 0000000..903b870 --- /dev/null +++ b/src/main2.cpp @@ -0,0 +1,136 @@ +/************************/ +/* main2.cpp */ +/* Version 1.0 */ +/* 2023/02/05 */ +/************************/ + +#include +#include +#include +#include +#include +#include +#include "algebra/matrix.h" +#include "hdf5/hdf5_ext.h" +#include "log/log.h" +#include "ann_mlp_sgd_v1.h" +#include "mnist.h" + +constexpr const char* const IMAGESDIR = "./data/MNIST"; +constexpr const char* const CONFIGFILE = "./build/config/nn2.hd5"; + +int main(int argc, char** argv) +{ + bool doTraining = true; + bool trainingFromStart = true; + + if (argc > 1) + { + std::string arg = argv[1]; + if (arg == "--training_start") + { + doTraining = true; + trainingFromStart = true; + } + else if (arg == "--training_continue") + { + doTraining = true; + trainingFromStart = false; + } + else if (arg == "--testing") { doTraining = false; } + } + + bool bFileLog = false; + LOGGER_PARAM(logging::LEVELMAX, logging::INFO); + LOGGER_PARAM(logging::LOGLINE, true); + LOGGER_PARAM(logging::LOGTIME, true); + if (bFileLog) + { + LOGGER_PARAM(logging::FILENAME, "out2.log"); + LOGGER_PARAM(logging::FILEOUT, true); + } + + if (doTraining) + LOGGER(logging::INFO) << std::string("*** Training mode") + (trainingFromStart ? " from start" : " continue"); + else LOGGER(logging::INFO) << std::string("*** Testing mode"); + + if (!std::filesystem::exists(CONFIGFILE)) + throw std::runtime_error(std::string("File: ").append(CONFIGFILE).append(" not found. Exiting...")); + h5::H5ppReader h5(CONFIGFILE); + std::string nname, archiveFile; + h5.read("nn2/nname", nname); + h5.read("nn2/data_file", archiveFile); + + if (doTraining) + { + std::string current_set; + h5.read("nn2/current_set", current_set); + std::unique_ptr> nn1; + if (trainingFromStart) + { + std::vector nnsize; + h5.read("nn2/" + current_set + "/size", nnsize); + nn1 = std::make_unique>(nnsize); + // nn1 = std::make_unique>(std::vector{784, 30, 10}); + // nn1 = std::make_unique>(std::vector{784, 64, 16, 10}); + nn1->SetName(nname); + } + else + { + nn1 = std::make_unique>(); + nn1->SetName(nname); + nn1->Deserialize(archiveFile); + } + size_t nEpochs, miniBatchSize; + double eta; + h5.read("nn2/" + current_set + "/nEpochs", nEpochs); + h5.read("nn2/" + current_set + "/miniBatchSize", miniBatchSize); + h5.read("nn2/" + current_set + "/eta", eta); + + nn::MNIST imgTrain = nn::MNIST(IMAGESDIR, true, false); + std::vector> images; + std::vector> labels; + for (const auto& img : imgTrain.Images()) + { + std::vector fimg(img.begin(), img.end()); + images.push_back(fimg); + } + for (const auto& lbl : imgTrain.Labels()) + { + std::vector flbl(lbl.begin(), lbl.end()); + labels.push_back(flbl); + } + + nn1->TrainSGD(images, labels, nEpochs, miniBatchSize, eta); + // nn1->TrainSGD(images, labels, 5, 10, 3.0); + nn1->Serialize(archiveFile); + LOGGER(logging::INFO) << std::string("*** Training completed"); + } + else + { + nn::MNIST imgTest = nn::MNIST(IMAGESDIR, false, false); + nn::ANN_MLP_SGD nn2; + nn2.SetName(nname); + nn2.Deserialize(archiveFile); + std::vector> images; + std::vector> labels; + for (const auto& img : imgTest.Images()) + { + std::vector fimg(img.begin(), img.end()); + images.push_back(fimg); + } + for (const auto& lbl : imgTest.Labels()) + { + std::vector flbl(lbl.begin(), lbl.end()); + labels.push_back(flbl); + } + int correct = nn2.TestSGD(images, labels); + + const int size = (int)imgTest.Images().size(); + LOGGER(logging::INFO) << (std::string("*** Correct: ") + std::to_string(correct) + std::string(" / ") + + std::to_string(size) + " (" + + std::to_string(100.0 * static_cast(correct) / size) + " %) ***"); + } + + return 0; +} diff --git a/src/main3.cpp b/src/main3.cpp new file mode 100644 index 0000000..ec3e7d8 --- /dev/null +++ b/src/main3.cpp @@ -0,0 +1,131 @@ +/************************/ +/* main3.cpp */ +/* Version 1.0 */ +/* 2023/02/19 */ +/************************/ + +#include +#include +#include +#include +#include +#include +#include "hdf5/hdf5_ext.h" +#include "log/log.h" +#include "ann_mlp_ga_v1.h" +#include "mnist.h" + +constexpr const char* const IMAGESDIR = "./data/MNIST"; +constexpr const char* const CONFIGFILE = "./build/config/nn3.hd5"; + +int main(int argc, char** argv) +{ + bool doTraining = true; + bool trainingFromStart = true; + + if (argc > 1) + { + std::string arg = argv[1]; + if (arg == "--training_start") + { + doTraining = true; + trainingFromStart = true; + } + else if (arg == "--training_continue") + { + doTraining = true; + trainingFromStart = false; + } + else if (arg == "--testing") { doTraining = false; } + } + + bool bFileLog = false; + LOGGER_PARAM(logging::LEVELMAX, logging::INFO); + LOGGER_PARAM(logging::LOGLINE, true); + LOGGER_PARAM(logging::LOGTIME, true); + if (bFileLog) + { + LOGGER_PARAM(logging::FILENAME, "out3.log"); + LOGGER_PARAM(logging::FILEOUT, true); + } + + if (doTraining) + LOGGER(logging::INFO) << std::string("*** Training mode") + (trainingFromStart ? " from start" : " continue"); + else LOGGER(logging::INFO) << std::string("*** Testing mode"); + + if (!std::filesystem::exists(CONFIGFILE)) + throw std::runtime_error(std::string("File: ").append(CONFIGFILE).append(" not found. Exiting...")); + h5::H5ppReader h5(CONFIGFILE); + std::string nname, archiveFile; + h5.read("nn3/nname", nname); + h5.read("nn3/data_file", archiveFile); + + if (doTraining) + { + std::string current_set; + h5.read("nn3/current_set", current_set); + std::unique_ptr> nn1; + if (trainingFromStart) + { + std::vector nnsize; + h5.read("nn3/" + current_set + "/size", nnsize); + nn1 = std::make_unique>(nnsize); + // nn1 = std::make_unique>(std::vector{784, 30, 10}); + nn1->SetName(nname); + } + else + { + nn1 = std::make_unique>(); + nn1->SetName(nname); + nn1->Deserialize(archiveFile); + } + size_t nGenerations, BatchSize; + h5.read("nn3/" + current_set + "/nGenerations", nGenerations); + h5.read("nn3/" + current_set + "/BatchSize", BatchSize); + + nn::MNIST imgTrain = nn::MNIST(IMAGESDIR, true, false); + std::vector> images; + std::vector> labels; + for (const auto& img : imgTrain.Images()) + { + std::vector fimg(img.begin(), img.end()); + images.push_back(fimg); + } + for (const auto& lbl : imgTrain.Labels()) + { + std::vector flbl(lbl.begin(), lbl.end()); + labels.push_back(flbl); + } + nn1->SetMixed(true); + nn1->TrainGA(images, labels, nGenerations, BatchSize, true); + // nn1->TrainGA(images, labels, 50, 200, true); + nn1->Serialize(archiveFile); + LOGGER(logging::INFO) << std::string("*** Training completed"); + } + else + { + nn::MNIST imgTest = nn::MNIST(IMAGESDIR, false, false); + nn::ANN_MLP_GA nn2; + nn2.SetName(nname); + nn2.Deserialize(archiveFile); + std::vector> images; + std::vector> labels; + for (const auto& img : imgTest.Images()) + { + std::vector fimg(img.begin(), img.end()); + images.push_back(fimg); + } + for (const auto& lbl : imgTest.Labels()) + { + std::vector flbl(lbl.begin(), lbl.end()); + labels.push_back(flbl); + } + int correct = nn2.TestGA(images, labels); + const int size = (int)imgTest.Images().size(); + LOGGER(logging::INFO) << (std::string("*** Correct: ") + std::to_string(correct) + std::string(" / ") + + std::to_string(size) + " (" + + std::to_string(100.0 * static_cast(correct) / size) + " %) ***"); + } + + return 0; +} diff --git a/src/mnist.cpp b/src/mnist.cpp new file mode 100644 index 0000000..22e5a08 --- /dev/null +++ b/src/mnist.cpp @@ -0,0 +1,146 @@ +/************************/ +/* mnist.cpp */ +/* Version 1.0 */ +/* 2023/02/05 */ +/************************/ + +#include +#include +#include +#include +#include "mnist.h" + +namespace fs = std::filesystem; + +nn::MNIST::MNIST(const std::string dirname, bool isTraining, bool bVerbose) + : sTrainingImages(dirname + "/train-images-idx3-ubyte"), sTrainingLabels(dirname + "/train-labels-idx1-ubyte"), + sTestingImages(dirname + "/t10k-images-idx3-ubyte"), sTestingLabels(dirname + "/t10k-labels-idx1-ubyte") +{ + if (!fs::exists(dirname)) + throw std::runtime_error(std::string("Directory: ").append(dirname).append(" not found. Exiting...")); +#ifdef DEBUG + if (!fs::exists(sTrainingImages)) + throw std::runtime_error(std::string("File: ").append(sTrainingImages).append(" not found. Exiting...")); + if (!fs::exists(sTrainingLabels)) + throw std::runtime_error(std::string("File: ").append(sTrainingLabels).append(" not found. Exiting...")); + if (!fs::exists(sTestingImages)) + throw std::runtime_error(std::string("File: ").append(sTestingImages).append(" not found. Exiting...")); + if (!fs::exists(sTestingLabels)) + throw std::runtime_error(std::string("File: ").append(sTestingLabels).append(" not found. Exiting...")); +#endif + if (isTraining) + { + ifImages.open(sTrainingImages, std::ios::in | std::ios::binary); + ifLabels.open(sTrainingLabels, std::ios::in | std::ios::binary); + } + else + { + ifImages.open(sTestingImages, std::ios::in | std::ios::binary); + ifLabels.open(sTestingLabels, std::ios::in | std::ios::binary); + } + + // Read the magic and the meta data + uint32_t magic_; + uint32_t nLabels_; + uint32_t rows_; + uint32_t cols_; + + ifImages.read(reinterpret_cast(&magic_), 4); + magic_ = SwapEndian(magic_); + if (magic_ != 2051) + throw std::runtime_error( + std::string("Incorrect images file magic: ").append(std::to_string(magic_)).append(". Exiting...")); + + ifLabels.read(reinterpret_cast(&magic_), 4); + magic_ = SwapEndian(magic_); + if (magic_ != 2049) + throw std::runtime_error( + std::string("Incorrect labels file magic: ").append(std::to_string(magic_)).append(". Exiting...")); + + ifImages.read(reinterpret_cast(&nItems), 4); + nItems = SwapEndian(nItems); + ifLabels.read(reinterpret_cast(&nLabels_), 4); + nLabels_ = SwapEndian(nLabels_); + + ifImages.read(reinterpret_cast(&rows_), 4); + rows_ = SwapEndian(rows_); + ifImages.read(reinterpret_cast(&cols_), 4); + cols_ = SwapEndian(cols_); + + assert((nItems == 60000) || (nItems == 10000)); + assert((nLabels_ == 60000) || (nLabels_ == 10000)); + assert(nItems == nLabels_); + assert(rows_ == DIM1); + assert(cols_ == DIM2); + + if (bVerbose) + { + std::cout << "Image and label num is: " << nItems << std::endl; + std::cout << "Image rows: " << DIM1 << ", cols: " << DIM2 << std::endl; + } + + // read the images + vLabelsRaw.resize(nItems); + for (size_t i = 0; i < nItems; ++i) + { + vImagesRaw.emplace_back(std::vector(DIMS)); + vImages.emplace_back(std::vector(DIMS)); + vLabels.emplace_back(std::vector(10, 0)); + } + + for (size_t i = 0; i < nItems; ++i) + { + // read image pixel + ifImages.read(&vImagesRaw[i][0], DIMS); + for (size_t k = 0; k < DIM2; ++k) + for (size_t j = 0; j < DIM1; ++j) vImages[i][k * DIM1 + j] = (vImagesRaw[i][k * DIM1 + j] == 0) ? 0 : 1; + // read label + ifLabels.read(&vLabelsRaw[i], 1); + vLabels[i][size_t(vLabelsRaw[i])] = 1; + } +} + +uint32_t nn::MNIST::SwapEndian(uint32_t val) +{ + uint32_t r = val; + char* buf = reinterpret_cast(&r); + std::swap(buf[0], buf[3]); + std::swap(buf[1], buf[2]); + return r; +} + +std::string nn::MNIST::ImageRaw(const size_t n) +{ + std::string ret{}; + for (size_t i = 0; i < DIMS; ++i) ret.append(std::string(1, vImagesRaw[n][i])); + return ret; +} + +size_t nn::MNIST::LabelNumeric(const size_t n) +{ + return static_cast(vLabelsRaw[n]); +} + +std::string nn::MNIST::Label(const size_t n) +{ + std::string ret; + ret = vLabelsRaw[n]; + return ret; +} + +void nn::MNIST::PrintImage(size_t n) +{ + for (size_t j = 0; j < DIM2; ++j) + { + for (size_t i = 0; i < DIM1; ++i) { std::cout << vImages[n][j * DIM1 + i]; } + std::cout << std::endl; + } +} + +void nn::MNIST::PrintLabel(size_t n) +{ + const auto& vec = vLabels[n]; + const auto printN = std::min((size_t)10, vec.size()); + for (size_t i = 0; i < printN; ++i) { std::cout << vec[i]; } + std::cout << std::endl; +} diff --git a/src/mnist.h b/src/mnist.h new file mode 100644 index 0000000..89217ff --- /dev/null +++ b/src/mnist.h @@ -0,0 +1,54 @@ +#ifndef _MNIST_H_122FDE746A84469CAF3080DBA4C76C11_ +#define _MNIST_H_122FDE746A84469CAF3080DBA4C76C11_ + +/************************/ +/* mnist.h */ +/* Version 1.0 */ +/* 2023/02/05 */ +/************************/ + +#include +#include +#include +#include + +namespace nn +{ + class MNIST + { + // images are 28 x 28 pixels + static constexpr size_t DIM1 = 28; + static constexpr size_t DIM2 = 28; + static constexpr size_t DIMS = 784; + + public: + MNIST(const std::string dirname, bool isTraining = true, bool bVerbose = false); + + std::vector>& Images() { return vImages; } + + std::vector>& Labels() { return vLabels; } + + std::string ImageRaw(const size_t n); + std::string Label(const size_t n); + size_t LabelNumeric(const size_t n); + void PrintImage(size_t n); + void PrintLabel(size_t n); + + private: + uint32_t SwapEndian(uint32_t val); + const std::string sTrainingImages; + const std::string sTrainingLabels; + const std::string sTestingImages; + const std::string sTestingLabels; + uint32_t nItems; + std::ifstream ifImages; + std::ifstream ifLabels; + std::vector> vImagesRaw; + std::vector> vImages; + std::vector vLabelsRaw; + std::vector> vLabels; + }; + +} // namespace nn + +#endif