diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 2b39a7204b..185aaf1dd8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -2,8 +2,6 @@ name: Builds on: - schedule: - - cron: 0 4 * * * workflow_call: inputs: gcc: @@ -18,6 +16,18 @@ on: type: string description: Machines to build for default: all + gcc_exceptions: + type: string + description: Exception groups for gcc + default: none + clang_exceptions: + type: string + description: Exception groups for clang + default: none + dry_run: + type: boolean + description: Print build matrix and exit + default: false verbose: type: boolean description: Show error outputs @@ -40,6 +50,18 @@ on: type: string description: Machines to build for (comma-separated | all) default: all + gcc_exceptions: + type: string + description: Exception groups for gcc (comma-separated and semi-colon delimited | none) + default: none + clang_exceptions: + type: string + description: Exception groups for clang (comma-separated and semi-colon delimited | none) + default: none + dry_run: + type: boolean + description: Print build matrix and exit + default: false verbose: type: boolean description: Show error outputs @@ -65,6 +87,10 @@ jobs: - name: Build command line args run: | ARGS="" + # dry-run + if [ "${{ inputs.dry_run }}" == "true" ]; then + ARGS="$ARGS --dry-run" + fi # verbose if [ "${{ inputs.verbose }}" == "true" ]; then ARGS="$ARGS --verbose" @@ -81,6 +107,14 @@ jobs: if [ ! -z "${{ inputs.gcc }}" ] && [ "${{ inputs.gcc }}" != "all" ]; then ARGS="$ARGS --gcc-versions ${{ inputs.gcc }}" fi + # exceptions + if [ ! -z "${{ inputs.gcc_exceptions }}" ] && [ "${{ inputs.gcc_exceptions }}" != "none" ]; then + no_spaces=$(tr -d '[:space:]' <<< "${{ inputs.gcc_exceptions }}") + IFS=';' read -r -a exceptions <<< "$no_spaces" + for exception in ${exceptions[@]}; do + ARGS="$ARGS --gcc-except $exception" + done + fi echo "BUILD_ARGS=$ARGS" >> $GITHUB_ENV - name: Run gcc builds @@ -100,6 +134,10 @@ jobs: - name: Build command line args run: | ARGS="" + # dry-run + if [ "${{ inputs.dry_run }}" == "true" ]; then + ARGS="$ARGS --dry-run" + fi # verbose if [ "${{ inputs.verbose }}" == "true" ]; then ARGS="$ARGS --verbose" @@ -116,6 +154,14 @@ jobs: if [ ! -z "${{ inputs.clang }}" ] && [ "${{ inputs.clang }}" != "all" ]; then ARGS="$ARGS --clang-versions ${{ inputs.clang }}" fi + # exceptions + if [ ! -z "${{ inputs.clang_exceptions }}" ] && [ "${{ inputs.clang_exceptions }}" != "none" ]; then + no_spaces=$(tr -d '[:space:]' <<< "${{ inputs.clang_exceptions }}") + IFS=';' read -r -a exceptions <<< "$no_spaces" + for exception in ${exceptions[@]}; do + ARGS="$ARGS --clang-except $exception" + done + fi echo "BUILD_ARGS=$ARGS" >> $GITHUB_ENV - name: Run clang builds diff --git a/.github/workflows/on_nightly.yml b/.github/workflows/on_nightly.yml index e17bc57f2d..be90e02542 100644 --- a/.github/workflows/on_nightly.yml +++ b/.github/workflows/on_nightly.yml @@ -3,14 +3,28 @@ on: workflow_dispatch: schedule: - cron: 30 11 * * * + - cron: 0 3 * * * jobs: tests: uses: ./.github/workflows/coverage_report_clusterfuzz.yml secrets: inherit + if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '30 11 * * *' }} coverage-report: uses: ./.github/workflows/coverage_report.yml secrets: inherit + if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '30 11 * * *' }} codeql: uses: ./.github/workflows/codeql.yml permissions: security-events: write + if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '30 11 * * *' }} + builds: + uses: ./.github/workflows/builds.yml + with: + # compiler,machine,target + gcc_exceptions: | + ALL,linux_gcc_power9,ALL; + ALL,linux_gcc_arm_n1,ALL; + gcc-10.5.0,linux_gcc_zen4,ALL; + gcc-11.4.0,linux_gcc_zen4,ALL + if: ${{ github.event_name == 'workflow_dispatch' || github.event.schedule == '0 3 * * *' }} diff --git a/contrib/build.sh b/contrib/build.sh index 97e96042e4..08369810bb 100755 --- a/contrib/build.sh +++ b/contrib/build.sh @@ -12,6 +12,7 @@ help() { echo " --no-clang Do not run any clang builds" echo " --no-deps Do not install deps during any builds" echo " --no-rust Do not install rust" + echo " --dry-run Print build matrix and exit" echo " --verbose Show output from failed builds" echo " --exit-on-err Exit upon hitting the first failed build" echo " --help -h Show this message and exit" @@ -28,6 +29,14 @@ help() { echo " --clang-versions -c Clang compiler versions to use for builds" echo " These should present on the host under /opt/clang" echo " For example: clang-15.0.6,clang-17.0.6" + echo " --gcc-except Comma separated group of gcc-compiler,machine,target to exclude" + echo " Use 'ALL' to exclude for all of any of the types." + echo " This argument can be supplied multiple times." + echo " For example: gcc-11.4.0,linux_gcc_zen4,ALL" + echo " --clang-except Comma separated group of clang-compiler,machine,target to exclude" + echo " Use 'ALL' to exclude for all of any of the types." + echo " For example: ALL,linux_clang_minimal,fdctl" + echo " This argument can be supplied multiple times." echo echo " Exit Codes:" echo " 0 all builds that ran were successful" @@ -39,11 +48,11 @@ help() { # helper functions inf() { - printf "[INFO] $@" + printf "[INFO] $*" } err() { - printf "[ERROR] $@" + printf "[ERROR] $*" } # Calculates the elapsed time @@ -52,7 +61,7 @@ err() { elapsed() { local start=$1 local stop=$2 - local diff=$(($stop-$start)) + local diff=$((stop-start)) local min=$((10#$diff / 60)) local sec=$((10#$diff % 60)) printf "%d:%02d" "$min" "$sec"; @@ -65,14 +74,16 @@ finish() { local CODE=$1 local STOP=$(date +%s) inf "Total Elapsed Time: " - elapsed $START $STOP + elapsed "$START" "$STOP" echo - rm -f $LOG_FILE - exit $CODE + rm -f "$LOG_FILE" + exit "$CODE" } GCC=() +GCC_EXCEPT=() CLANG=() +CLANG_EXCEPT=() TARGETS=() MACHINES=() @@ -96,6 +107,10 @@ while [[ $# -gt 0 ]]; do "--no-rust") NO_RUST=1 ;; + # print build matrix and exit + "--dry-run") + DRY_RUN=1 + ;; # exit upon hitting the first error "--exit-on-err") EXIT_ON_ERR=1 @@ -124,6 +139,16 @@ while [[ $# -gt 0 ]]; do IFS=',' read -r -a CLANG <<< "$1" shift 1 ;; + # GCC compiler exception groups + "--gcc-except") + GCC_EXCEPT+=("$1") + shift 1 + ;; + # Clang compiler exception groups + "--clang-except") + CLANG_EXCEPT+=("$1") + shift 1 + ;; "-h"|"--help") help exit 0 @@ -207,10 +232,64 @@ echo "Starting Build Matrix..." echo "*************************" echo "machines=[ ${MACHINES[*]} ]" echo "clang=[ ${CLANG[*]} ]" +echo "clang_except=[ ${CLANG_EXCEPT[*]} ]" echo "gcc=[ ${GCC[*]} ]" +echo "gcc_except=[ ${GCC_EXCEPT[*]} ]" echo "targets=[ ${TARGETS[*]} ]" echo +if [[ $DRY_RUN -eq 1 ]]; then + exit 0 +fi + +skip_compiler() { + local compiler=$1 + local compiler_type="${compiler%-*}" + local except_list="${compiler_type^^}_EXCEPT[@]" + for except in "${!except_list}"; do + C="$(cut -d',' -f1 <<<"$except")" + M="$(cut -d',' -f2 <<<"$except")" + T="$(cut -d',' -f3 <<<"$except")" + if [[ "$C" == "$compiler" ]] && [[ "$M" == "ALL" ]] && [[ "$T" == "ALL" ]]; then + return 0 + fi + done + return 1 +} + +skip_compiler_machine() { + local compiler=$1 + local machine=$2 + local compiler_type="${compiler%-*}" + local except_list="${compiler_type^^}_EXCEPT[@]" + for except in "${!except_list}"; do + C="$(cut -d',' -f1 <<<"$except")" + M="$(cut -d',' -f2 <<<"$except")" + T="$(cut -d',' -f3 <<<"$except")" + if [[ "$C" == "$compiler" || "$C" == "ALL" ]] && [[ "$M" == "$machine" ]] && [[ "$T" == "ALL" ]]; then + return 0 + fi + done + return 1 +} + +skip_compiler_machine_target() { + local compiler=$1 + local machine=$2 + local target=$3 + local compiler_type="${compiler%-*}" + local except_list="${compiler_type^^}_EXCEPT[@]" + for except in "${!except_list}"; do + C="$(cut -d',' -f1 <<<"$except")" + M="$(cut -d',' -f2 <<<"$except")" + T="$(cut -d',' -f3 <<<"$except")" + if [[ "$C" == "$compiler" || "$C" == "ALL" ]] && [[ "$M" == "$machine" || "$M" == "ALL" ]] && [[ "$T" == "$target" ]]; then + return 0 + fi + done + return 1 +} + # Install rust and packages, also fetch the git repositories # needed for compiling. @@ -227,6 +306,11 @@ if [[ $NO_GCC -ne 1 ]]; then GCC_START=$(date +%s) inf "Starting gcc builds...\n\n" for compiler in "${GCC[@]}"; do + # Should we skip this compiler? + if skip_compiler "$compiler"; then + inf "Skipping all machines and targets for - $compiler\n" + continue + fi if [[ ! -f /opt/gcc/$compiler/activate ]]; then err "Environment activate script not found at /opt/gcc/$compiler... exiting.\n" finish 2 @@ -238,11 +322,11 @@ if [[ $NO_GCC -ne 1 ]]; then start=$(date +%s) inf "Installing dependencies with $compiler...\n" ./deps.sh nuke > /dev/null 2>&1 - CC=gcc CXX=g++ ./deps.sh +dev fetch install > $LOG_FILE 2>&1 + CC=gcc CXX=g++ ./deps.sh +dev fetch install > "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then err "Failed to install deps with $compiler... exiting.\n" if [[ $VERBOSE -eq 1 ]]; then - cat $LOG_FILE + cat "$LOG_FILE" fi if [[ $EXIT_ON_ERR -eq 1 ]]; then finish 3 @@ -252,11 +336,16 @@ if [[ $NO_GCC -ne 1 ]]; then fi stop=$(date +%s) inf "Elapsed Time: " - elapsed $start $stop + elapsed "$start" "$stop" echo fi for machine in "${MACHINES[@]}"; do MACHINE="${machine%.mk}" + # Should we skip this compiler+machine? + if skip_compiler_machine "$compiler" "$MACHINE"; then + inf "Skipping all targets for - $compiler $MACHINE\n" + continue + fi if [[ "$MACHINE" != *"clang"* ]]; then # override any targets list with supplied --targets BUILD_TARGETS=() @@ -277,22 +366,27 @@ if [[ $NO_GCC -ne 1 ]]; then # Truncate the LOG_FILE before every build (not every target) # so that if multiple targets fail, we can capture all the # errors that occur. - >$LOG_FILE + > "$LOG_FILE" # We compile each target separately and record if it # fails so that we can list exactly which targets failed. # The output is redirected to the LOG_FILE and we only # print it to stdout if there's a failure and --verbose # is specified. This keeps our output compact and readable. for target in "${BUILD_TARGETS[@]}"; do - MACHINE=${MACHINE} CC=gcc make -j $target >> $LOG_FILE 2>&1 + # Should we skip this compiler+machine+target? + if skip_compiler_machine_target "$compiler" "$MACHINE" "$target"; then + inf "Skipping - $compiler $MACHINE $target\n" + continue + fi + MACHINE=${MACHINE} CC=gcc make -j "$target" >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then - FAILED+=( $target ) + FAILED+=( "$target" ) FAIL=1 fi done stop=$(date +%s) inf "Done... Elapsed Time: " - elapsed $start $stop + elapsed "$start" "$stop" echo if [[ ${#FAILED[@]} -gt 0 ]]; then err "Failed Targets: " @@ -302,10 +396,10 @@ if [[ $NO_GCC -ne 1 ]]; then echo " ./deps.sh nuke" echo " FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev fetch check install" echo " make -j distclean" - echo " MACHINE=${MACHINE} CC=gcc make -j ${FAILED[@]}" + echo " MACHINE=${MACHINE} CC=gcc make -j ${FAILED[*]}" if [[ $VERBOSE -eq 1 ]]; then err "Failure Logs:\n" - cat $LOG_FILE + cat "$LOG_FILE" fi if [[ $EXIT_ON_ERR -eq 1 ]]; then finish 1 @@ -321,7 +415,7 @@ if [[ $NO_GCC -ne 1 ]]; then done GCC_STOP=$(date +%s) inf "Done with gcc builds in " - elapsed $GCC_START $GCC_STOP + elapsed "$GCC_START" "$GCC_STOP" echo fi @@ -329,6 +423,11 @@ if [[ $NO_CLANG -ne 1 ]]; then CLANG_START=$(date +%s) inf "Starting clang builds...\n\n" for compiler in "${CLANG[@]}"; do + # Should we skip this compiler? + if skip_compiler "$compiler"; then + inf "Skipping all machines and targets for - $compiler\n" + continue + fi if [[ ! -f /opt/clang/$compiler/activate ]]; then err "Environment activate script not found at /opt/clang/$compiler... exiting.\n" finish 2 @@ -340,11 +439,11 @@ if [[ $NO_CLANG -ne 1 ]]; then start=$(date +%s) inf "Installing dependencies with $compiler...\n" ./deps.sh nuke > /dev/null 2>&1 - CC=clang CXX=clang++ ./deps.sh +dev fetch install > $LOG_FILE 2>&1 + CC=clang CXX=clang++ ./deps.sh +dev fetch install > "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then err "Failed to install deps with $compiler...\n" if [[ $VERBOSE -eq 1 ]]; then - cat $LOG_FILE + cat "$LOG_FILE" fi if [[ $EXIT_ON_ERR -eq 1 ]]; then finish 3 @@ -354,11 +453,16 @@ if [[ $NO_CLANG -ne 1 ]]; then fi stop=$(date +%s) inf "Elapsed Time: " - elapsed $start $stop + elapsed "$start" "$stop" echo fi for machine in "${MACHINES[@]}"; do MACHINE="${machine%.mk}" + # Should we skip this compiler+machine? + if skip_compiler_machine "$compiler" "$MACHINE"; then + inf "Skipping all targets for - $compiler $MACHINE\n" + continue + fi if [[ "$MACHINE" != *"gcc"* ]]; then # override any targets list with supplied --targets BUILD_TARGETS=() @@ -379,22 +483,27 @@ if [[ $NO_CLANG -ne 1 ]]; then # Truncate the LOG_FILE before every build (not every target) # so that if multiple targets fail, we can capture all the # errors that occur. - >$LOG_FILE + > "$LOG_FILE" # We compile each target separately and record if it # fails so that we can list exactly which targets failed. # The output is redirected to the LOG_FILE and we only # print it to stdout if there's a failure and --verbose # is specified. This keeps our output compact and readable. for target in "${BUILD_TARGETS[@]}"; do - MACHINE=${MACHINE} CC=clang make -j $target >> $LOG_FILE 2>&1 + # Should we skip this compiler+machine+target? + if skip_compiler_machine_target "$compiler" "$MACHINE" "$target"; then + inf "Skipping - $compiler $MACHINE $target\n" + continue + fi + MACHINE=${MACHINE} CC=clang make -j "$target" >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then - FAILED+=( $target ) + FAILED+=( "$target" ) FAIL=1 fi done stop=$(date +%s) inf "Done... Elapsed Time: " - elapsed $start $stop + elapsed "$start" "$stop" echo if [[ ${#FAILED[@]} -gt 0 ]]; then err "Failed Targets: " @@ -403,10 +512,10 @@ if [[ $NO_CLANG -ne 1 ]]; then echo " ./deps.sh nuke" echo " FD_AUTO_INSTALL_PACKAGES=1 ./deps.sh +dev fetch check install" echo " make -j distclean" - echo " MACHINE=${MACHINE} CC=clang make -j ${FAILED[@]}" + echo " MACHINE=${MACHINE} CC=clang make -j ${FAILED[*]}" if [[ $VERBOSE -eq 1 ]]; then err "Failure Logs:\n" - cat $LOG_FILE + cat "$LOG_FILE" fi if [[ $EXIT_ON_ERR -eq 1 ]]; then finish 1 @@ -422,7 +531,7 @@ if [[ $NO_CLANG -ne 1 ]]; then done CLANG_STOP=$(date +%s) inf "Done with clang builds in " - elapsed $CLANG_START $CLANG_STOP + elapsed "$CLANG_START" "$CLANG_STOP" echo fi diff --git a/src/flamenco/nanopb/README.txt b/src/flamenco/nanopb/README.txt index 48b84fd566..990b62e193 100644 --- a/src/flamenco/nanopb/README.txt +++ b/src/flamenco/nanopb/README.txt @@ -1,13 +1,10 @@ -This directory contains a copy of the Nanopb library. +This directory contains a copy of the Nanopb library C source code. -This library is copied exactly from commit `839156b`, and there are +This library is copied exactly from tag "nanopb-0.4.9.1", and there are no Firedancer specific modifications to the code. You should not make local modifications to nanopb. Instead prefer to upstream changes, or make a local patch if needed. -For licensing information, refer to NOTICE in the root of this repo. +Use `fetch.sh` and `nanopb_tag.txt` to update. -nanopb_generator.py can be invoked like so: - - cd src/flamenco/types - nanopb_generator.py -L "" fd_solana_block.proto +For licensing information, refer to NOTICE in the root of this repo. \ No newline at end of file diff --git a/src/flamenco/nanopb/fetch.sh b/src/flamenco/nanopb/fetch.sh new file mode 100755 index 0000000000..fa44e6411d --- /dev/null +++ b/src/flamenco/nanopb/fetch.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -x + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +REPO_URL="https://raw.githubusercontent.com/nanopb/nanopb" +REPO_TAG=$(cat nanopb_tag.txt) + +FILES=( + "pb_common.h" + "pb_common.c" + "pb_decode.h" + "pb_decode.c" + "pb_encode.h" + "pb_encode.c" + "pb.h" +) + +for file in "${FILES[@]}"; do + RAW_URL="$REPO_URL/refs/tags/$REPO_TAG/$file" + curl -s -o $SCRIPT_DIR/$file $REPO_URL/$REPO_TAG/$file + # Replace #include "pb.h" with #include "pb_firedancer.h" + sed -i 's/#include "pb.h"/#include "pb_firedancer.h"/' $file +done \ No newline at end of file diff --git a/src/flamenco/nanopb/nanopb_tag.txt b/src/flamenco/nanopb/nanopb_tag.txt new file mode 100644 index 0000000000..e09f21edbb --- /dev/null +++ b/src/flamenco/nanopb/nanopb_tag.txt @@ -0,0 +1 @@ +nanopb-0.4.9.1 \ No newline at end of file diff --git a/src/flamenco/nanopb/pb.h b/src/flamenco/nanopb/pb.h index ef3d83e95a..10249bb651 100644 --- a/src/flamenco/nanopb/pb.h +++ b/src/flamenco/nanopb/pb.h @@ -65,7 +65,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION "nanopb-0.4.8-dev" +#define NANOPB_VERSION "nanopb-0.4.9.1" /* Include all the system headers needed by nanopb. You will need the * definitions of the following: @@ -215,12 +215,23 @@ PB_STATIC_ASSERT(1, STATIC_ASSERT_IS_NOT_WORKING) #endif #endif +/* Data type for storing encoded data and other byte streams. + * This typedef exists to support platforms where uint8_t does not exist. + * You can regard it as equivalent on uint8_t on other platforms. + */ +#if defined(PB_BYTE_T_OVERRIDE) +typedef PB_BYTE_T_OVERRIDE pb_byte_t; +#elif defined(UINT8_MAX) +typedef uint8_t pb_byte_t; +#else +typedef uint_least8_t pb_byte_t; +#endif + /* List of possible field types. These are used in the autogenerated code. * Least-significant 4 bits tell the scalar type * Most-significant 4 bits specify repeated/required/packed etc. */ - -typedef uint_least8_t pb_type_t; +typedef pb_byte_t pb_type_t; /**** Field data types ****/ @@ -301,12 +312,6 @@ typedef uint_least8_t pb_type_t; #endif #define PB_SIZE_MAX ((pb_size_t)-1) -/* Data type for storing encoded data and other byte streams. - * This typedef exists to support platforms where uint8_t does not exist. - * You can regard it as equivalent on uint8_t on other platforms. - */ -typedef uint_least8_t pb_byte_t; - /* Forward declaration of struct types */ typedef struct pb_istream_s pb_istream_t; typedef struct pb_ostream_s pb_ostream_t; diff --git a/src/flamenco/nanopb/pb_decode.c b/src/flamenco/nanopb/pb_decode.c index e0eb1a2218..0ed4841472 100644 --- a/src/flamenco/nanopb/pb_decode.c +++ b/src/flamenco/nanopb/pb_decode.c @@ -4,13 +4,14 @@ */ /* Use the GCC warn_unused_result attribute to check that all return values - * are propagated correctly. On other compilers and gcc before 3.4.0 just - * ignore the annotation. + * are propagated correctly. On other compilers, gcc before 3.4.0 and iar + * before 9.40.1 just ignore the annotation. */ -#if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) - #define checkreturn -#else +#if (defined(__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))) || \ + (defined(__IAR_SYSTEMS_ICC__) && (__VER__ >= 9040001)) #define checkreturn __attribute__((warn_unused_result)) +#else + #define checkreturn #endif #include "pb_firedancer.h" @@ -1167,7 +1168,7 @@ bool checkreturn pb_decode_ex(pb_istream_t *stream, const pb_msgdesc_t *fields, status = pb_decode_inner(&substream, fields, dest_struct, flags); if (!pb_close_string_substream(stream, &substream)) - return false; + status = false; } #ifdef PB_ENABLE_MALLOC diff --git a/src/flamenco/nanopb/pb_decode.h b/src/flamenco/nanopb/pb_decode.h index f33bca729d..367a4d7041 100644 --- a/src/flamenco/nanopb/pb_decode.h +++ b/src/flamenco/nanopb/pb_decode.h @@ -15,7 +15,7 @@ extern "C" { /* Structure for defining custom input streams. You will need to provide * a callback function to read the bytes from your storage, which can be * for example a file or a network socket. - * + * * The callback must conform to these rules: * * 1) Return false on IO errors. This will cause decoding to abort. @@ -37,10 +37,21 @@ struct pb_istream_s bool (*callback)(pb_istream_t *stream, pb_byte_t *buf, size_t count); #endif - void *state; /* Free field for use by callback implementation */ - size_t bytes_left; + /* state is a free field for use of the callback function defined above. + * Note that when pb_istream_from_buffer() is used, it reserves this field + * for its own use. + */ + void *state; + /* Maximum number of bytes left in this stream. Callback can report + * EOF before this limit is reached. Setting a limit is recommended + * when decoding directly from file or network streams to avoid + * denial-of-service by excessively long messages. + */ + size_t bytes_left; + #ifndef PB_NO_ERRMSG + /* Pointer to constant (ROM) string when decoding function returns error */ const char *errmsg; #endif }; @@ -54,7 +65,7 @@ struct pb_istream_s /*************************** * Main decoding functions * ***************************/ - + /* Decode a single protocol buffers message from input stream into a C structure. * Returns true on success, false on any failure. * The actual struct pointed to by dest must match the description in fields. @@ -65,7 +76,7 @@ struct pb_istream_s * MyMessage msg = {}; * uint8_t buffer[64]; * pb_istream_t stream; - * + * * // ... read some data into buffer ... * * stream = pb_istream_from_buffer(buffer, count); diff --git a/src/flamenco/nanopb/pb_encode.c b/src/flamenco/nanopb/pb_encode.c index 5574648a12..56067c68b3 100644 --- a/src/flamenco/nanopb/pb_encode.c +++ b/src/flamenco/nanopb/pb_encode.c @@ -8,13 +8,14 @@ #include "pb_common.h" /* Use the GCC warn_unused_result attribute to check that all return values - * are propagated correctly. On other compilers and gcc before 3.4.0 just - * ignore the annotation. + * are propagated correctly. On other compilers, gcc before 3.4.0 and iar + * before 9.40.1 just ignore the annotation. */ -#if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) - #define checkreturn -#else +#if (defined(__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))) || \ + (defined(__IAR_SYSTEMS_ICC__) && (__VER__ >= 9040001)) #define checkreturn __attribute__((warn_unused_result)) +#else + #define checkreturn #endif /************************************** diff --git a/src/flamenco/nanopb/pb_encode.h b/src/flamenco/nanopb/pb_encode.h index 891368322f..8aabaef0cb 100644 --- a/src/flamenco/nanopb/pb_encode.h +++ b/src/flamenco/nanopb/pb_encode.h @@ -6,7 +6,7 @@ #ifndef PB_ENCODE_H_INCLUDED #define PB_ENCODE_H_INCLUDED -#include "pb.h" +#include "pb_firedancer.h" #ifdef __cplusplus extern "C" { @@ -37,11 +37,21 @@ struct pb_ostream_s #else bool (*callback)(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); #endif - void *state; /* Free field for use by callback implementation. */ - size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ - size_t bytes_written; /* Number of bytes written so far. */ + + /* state is a free field for use of the callback function defined above. + * Note that when pb_ostream_from_buffer() is used, it reserves this field + * for its own use. + */ + void *state; + + /* Limit number of output bytes written. Can be set to SIZE_MAX. */ + size_t max_size; + + /* Number of bytes written so far. */ + size_t bytes_written; #ifndef PB_NO_ERRMSG + /* Pointer to constant (ROM) string when decoding function returns error */ const char *errmsg; #endif }; diff --git a/src/flamenco/runtime/program/fd_stake_program.c b/src/flamenco/runtime/program/fd_stake_program.c index 605446a541..856dbc119b 100644 --- a/src/flamenco/runtime/program/fd_stake_program.c +++ b/src/flamenco/runtime/program/fd_stake_program.c @@ -445,19 +445,24 @@ typedef fd_stake_history_entry_t fd_stake_activation_status_t; fd_stake_history_entry_t const * fd_stake_history_ele_query_const( fd_stake_history_t const * history, ulong epoch ) { - if( 0 == history->fd_stake_history_len ) + if( 0 == history->fd_stake_history_len ) { return NULL; + } - if( epoch > history->fd_stake_history[0].epoch ) + if( epoch > history->fd_stake_history[0].epoch ) { return NULL; + } ulong off = (history->fd_stake_history[0].epoch - epoch); - if( off >= history->fd_stake_history_len ) + if( off >= history->fd_stake_history_len ) { return NULL; + } ulong e = (off + history->fd_stake_history_offset) & (history->fd_stake_history_size - 1); - FD_TEST(history->fd_stake_history[e].epoch == epoch); + if ( history->fd_stake_history[e].epoch != epoch ) { + return NULL; + } return &history->fd_stake_history[e]; } diff --git a/src/flamenco/runtime/tests/fetch_and_generate.sh b/src/flamenco/runtime/tests/fetch_and_generate.sh index 7cbe34ae22..8947c710d3 100755 --- a/src/flamenco/runtime/tests/fetch_and_generate.sh +++ b/src/flamenco/runtime/tests/fetch_and_generate.sh @@ -1,7 +1,7 @@ #!/bin/bash -# This commit hash was taken from flamenco/nanopb/README.md -FD_NANOPB_COMMIT=839156b71c64b0a6073ad9c8d793f5913075d9bb +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +FD_NANOPB_TAG=$(cat ../../nanopb/nanopb_tag.txt) # Create venv and install packages python3.11 -m venv nanopb_venv @@ -12,13 +12,13 @@ pip install protobuf grpcio-tools if [ ! -d nanopb ]; then git clone --depth=1 -q https://github.com/nanopb/nanopb.git cd nanopb - git fetch --depth=1 -q origin $FD_NANOPB_COMMIT - git checkout -q $FD_NANOPB_COMMIT + git fetch --depth=1 -q origin $FD_NANOPB_TAG:refs/tags$FD_NANOPB_TAG + git checkout -q $FD_NANOPB_TAG cd .. else cd nanopb - git fetch --depth=1 -q origin $FD_NANOPB_COMMIT - git checkout -q $FD_NANOPB_COMMIT + git fetch --depth=1 -q origin $FD_NANOPB_TAG:refs/tags$FD_NANOPB_TAG + git checkout -q $FD_NANOPB_TAG cd .. fi diff --git a/src/flamenco/runtime/tests/generated/context.pb.c b/src/flamenco/runtime/tests/generated/context.pb.c index 8902c45119..ccf714ea12 100644 --- a/src/flamenco/runtime/tests/generated/context.pb.c +++ b/src/flamenco/runtime/tests/generated/context.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "context.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/context.pb.h b/src/flamenco/runtime/tests/generated/context.pb.h index 322448f003..28df3fb89d 100644 --- a/src/flamenco/runtime/tests/generated/context.pb.h +++ b/src/flamenco/runtime/tests/generated/context.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_CONTEXT_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_CONTEXT_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/elf.pb.c b/src/flamenco/runtime/tests/generated/elf.pb.c index d68b798e1a..2c0189651a 100644 --- a/src/flamenco/runtime/tests/generated/elf.pb.c +++ b/src/flamenco/runtime/tests/generated/elf.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "elf.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/elf.pb.h b/src/flamenco/runtime/tests/generated/elf.pb.h index 554878ff34..4dc0948ad2 100644 --- a/src/flamenco/runtime/tests/generated/elf.pb.h +++ b/src/flamenco/runtime/tests/generated/elf.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_ELF_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_ELF_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/invoke.pb.c b/src/flamenco/runtime/tests/generated/invoke.pb.c index 22c25ad9af..1653dfcef9 100644 --- a/src/flamenco/runtime/tests/generated/invoke.pb.c +++ b/src/flamenco/runtime/tests/generated/invoke.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "invoke.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/invoke.pb.h b/src/flamenco/runtime/tests/generated/invoke.pb.h index 1ec09e2b65..20076c94a8 100644 --- a/src/flamenco/runtime/tests/generated/invoke.pb.h +++ b/src/flamenco/runtime/tests/generated/invoke.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_INVOKE_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_INVOKE_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/metadata.pb.c b/src/flamenco/runtime/tests/generated/metadata.pb.c index 9332d18551..a8b78fb626 100644 --- a/src/flamenco/runtime/tests/generated/metadata.pb.c +++ b/src/flamenco/runtime/tests/generated/metadata.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "metadata.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/metadata.pb.h b/src/flamenco/runtime/tests/generated/metadata.pb.h index 30b7f9fad3..5b3fba1788 100644 --- a/src/flamenco/runtime/tests/generated/metadata.pb.h +++ b/src/flamenco/runtime/tests/generated/metadata.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_METADATA_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_METADATA_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/pack.pb.c b/src/flamenco/runtime/tests/generated/pack.pb.c index cec1410980..18071411b0 100644 --- a/src/flamenco/runtime/tests/generated/pack.pb.c +++ b/src/flamenco/runtime/tests/generated/pack.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "pack.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/pack.pb.h b/src/flamenco/runtime/tests/generated/pack.pb.h index fd973877b9..eccd3a737c 100644 --- a/src/flamenco/runtime/tests/generated/pack.pb.h +++ b/src/flamenco/runtime/tests/generated/pack.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_PACK_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_PACK_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/serialize.pb.c b/src/flamenco/runtime/tests/generated/serialize.pb.c index 022f0f0bfe..a5ac7ec575 100644 --- a/src/flamenco/runtime/tests/generated/serialize.pb.c +++ b/src/flamenco/runtime/tests/generated/serialize.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "serialize.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/serialize.pb.h b/src/flamenco/runtime/tests/generated/serialize.pb.h index b6071d0c65..f3ee537cab 100644 --- a/src/flamenco/runtime/tests/generated/serialize.pb.h +++ b/src/flamenco/runtime/tests/generated/serialize.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_SERIALIZE_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_SERIALIZE_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/shred.pb.c b/src/flamenco/runtime/tests/generated/shred.pb.c index e40c965cac..5f90d00318 100644 --- a/src/flamenco/runtime/tests/generated/shred.pb.c +++ b/src/flamenco/runtime/tests/generated/shred.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "shred.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/shred.pb.h b/src/flamenco/runtime/tests/generated/shred.pb.h index 9a89988769..899a789548 100644 --- a/src/flamenco/runtime/tests/generated/shred.pb.h +++ b/src/flamenco/runtime/tests/generated/shred.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_SHRED_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_SHRED_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/txn.pb.c b/src/flamenco/runtime/tests/generated/txn.pb.c index efce9a026f..58d9e5e9a4 100644 --- a/src/flamenco/runtime/tests/generated/txn.pb.c +++ b/src/flamenco/runtime/tests/generated/txn.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "txn.pb.h" #if PB_PROTO_HEADER_VERSION != 40 diff --git a/src/flamenco/runtime/tests/generated/txn.pb.h b/src/flamenco/runtime/tests/generated/txn.pb.h index ea13dfd1ef..d56a38562f 100644 --- a/src/flamenco/runtime/tests/generated/txn.pb.h +++ b/src/flamenco/runtime/tests/generated/txn.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_TXN_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_TXN_PB_H_INCLUDED diff --git a/src/flamenco/runtime/tests/generated/vm.pb.c b/src/flamenco/runtime/tests/generated/vm.pb.c index 0a0fedc7a3..539eedbad1 100644 --- a/src/flamenco/runtime/tests/generated/vm.pb.c +++ b/src/flamenco/runtime/tests/generated/vm.pb.c @@ -1,5 +1,5 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #include "vm.pb.h" #if PB_PROTO_HEADER_VERSION != 40 @@ -38,3 +38,4 @@ PB_BIND(FD_EXEC_TEST_RETURN_DATA, fd_exec_test_return_data_t, AUTO) + diff --git a/src/flamenco/runtime/tests/generated/vm.pb.h b/src/flamenco/runtime/tests/generated/vm.pb.h index 9d47eafafc..3caf7224b2 100644 --- a/src/flamenco/runtime/tests/generated/vm.pb.h +++ b/src/flamenco/runtime/tests/generated/vm.pb.h @@ -1,5 +1,5 @@ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.4.8-dev */ +/* Generated by nanopb-0.4.9.1 */ #ifndef PB_ORG_SOLANA_SEALEVEL_V1_VM_PB_H_INCLUDED #define PB_ORG_SOLANA_SEALEVEL_V1_VM_PB_H_INCLUDED diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c b/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c index 3ec43dc0b7..1a6dfad82f 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c +++ b/src/flamenco/vm/syscall/fd_vm_syscall_cpi.c @@ -615,6 +615,13 @@ fd_vm_syscall_cpi_check_authorized_program( fd_pubkey_t const * program_i /* Declare the size of the underlying data */ \ ulong FD_EXPAND_THEN_CONCAT2(decl, _len) = FD_EXPAND_THEN_CONCAT2(decl, _box)->len; +/* The data and lamports fields are in an Rc> in the Rust SDK AccountInfo */ +#define VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_RC_REFCELL_VADDR( vm, acc_info, decl ) \ + ulong decl = acc_info->lamports_box_addr; + +#define VM_SYSCALL_CPI_ACC_INFO_DATA_RC_REFCELL_VADDR( vm, acc_info, decl ) \ + ulong decl = acc_info->data_box_addr; + #define VM_SYSCALL_CPI_SET_ACC_INFO_DATA_LEN( vm, acc_info, decl, len_ ) \ FD_EXPAND_THEN_CONCAT2(decl, _box)->len = len_; @@ -641,4 +648,6 @@ fd_vm_syscall_cpi_check_authorized_program( fd_pubkey_t const * program_i #undef VM_SYSCALL_CPI_ACC_INFO_LAMPORTS #undef VM_SYSCALL_CPI_ACC_INFO_DATA #undef VM_SYSCALL_CPI_ACC_INFO_METADATA +#undef VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_RC_REFCELL_VADDR +#undef VM_SYSCALL_CPI_ACC_INFO_DATA_RC_REFCELL_VADDR #undef VM_SYSCALL_CPI_SET_ACC_INFO_DATA_LEN diff --git a/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c b/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c index ae5c105056..7d09c0e63e 100644 --- a/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c +++ b/src/flamenco/vm/syscall/fd_vm_syscall_cpi_common.c @@ -331,6 +331,30 @@ VM_SYSCALL_CPI_TRANSLATE_AND_UPDATE_ACCOUNTS_FUNC( } found = 1; + if ( vm->direct_mapping ) { + /* Check that the account's lamports Rc> is not stored in the account. Because a refcell is + only present if the Rust SDK is used, we only need to check this for the Rust SDK. + + https://github.com/anza-xyz/agave/blob/c79c3c9e67274594ed4f43ed8386e9ddc60a99e9/programs/bpf_loader/src/syscalls/cpi.rs#L140 */ + #ifdef VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_RC_REFCELL_VADDR + VM_SYSCALL_CPI_ACC_INFO_LAMPORTS_RC_REFCELL_VADDR( vm, (account_infos + j), lamports_rc_vaddr ) + if ( FD_UNLIKELY( lamports_rc_vaddr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + return FD_VM_SYSCALL_ERR_INVALID_POINTER; + } + #endif + + /* Check that the account's data Rc> is not stored in the account. Because a refcell is + only present if the Rust SDK is used, we only need to check this for the Rust SDK. + + https://github.com/anza-xyz/agave/blob/c79c3c9e67274594ed4f43ed8386e9ddc60a99e9/programs/bpf_loader/src/syscalls/cpi.rs#L161 */ + #ifdef VM_SYSCALL_CPI_ACC_INFO_DATA_RC_REFCELL_VADDR + VM_SYSCALL_CPI_ACC_INFO_DATA_RC_REFCELL_VADDR( vm, (account_infos + j), data_rc_vaddr ) + if ( FD_UNLIKELY( data_rc_vaddr >= FD_VM_MEM_MAP_INPUT_REGION_START ) ) { + return FD_VM_SYSCALL_ERR_INVALID_POINTER; + } + #endif + } + /* Update the callee account to reflect any changes the caller has made */ if( FD_UNLIKELY( VM_SYCALL_CPI_UPDATE_CALLEE_ACC_FUNC(vm, &account_infos[j], (uchar)instruction_accounts[i].index_in_caller ) ) ) { return 1001;