Merge branch 'main' into log_metricky_events

.azure-pipelines-templates/matrix.yml

@@ -69,20 +69,6 @@ parameters:
       ctest_args: '-L "raft_scenario"'
 
 jobs:
-  # Debug
-  - ${{ each target in parameters.target }}:
-      - template: common.yml
-        parameters:
-          target: ${{ target }}
-          env: ${{ parameters.env[target] }}
-          cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build[target].cmake_args }}"
-          cmake_env: "${{ parameters.build[target].cmake_env }}"
-          suffix: "Debug"
-          artifact_name: "${{ target }}_Debug"
-          ctest_filter: "${{ parameters.test[target].ctest_args }}"
-          depends_on: configure
-          installExtendedTestingTools: false
-
   # Tracing
   - template: common.yml
     parameters:
@@ -97,57 +83,6 @@ jobs:
       installExtendedTestingTools: false
       ninja_targets: "${{ parameters.build.tracing.ninja_targets }}"
 
-  # Performance
-  - ${{ if eq(parameters.perf_tests, 'run') }}:
-      - template: common.yml
-        parameters:
-          target: SGX
-          env: ${{ parameters.env.SGX }}
-          cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }}"
-          suffix: "Perf"
-          artifact_name: "SGX_Perf"
-          ctest_filter: "${{ parameters.test.perf.ctest_args }}"
-          depends_on: configure
-          installExtendedTestingTools: false
-
-  - ${{ if eq(parameters.perf_tests, 'run') }}:
-      - template: common.yml
-        parameters:
-          target: Virtual
-          env: ${{ parameters.env.Virtual }}
-          cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.Virtual.cmake_args }}"
-          cmake_env: "${{ parameters.build.Virtual.cmake_env }}"
-          suffix: "Perf"
-          artifact_name: "Virtual_Perf"
-          ctest_filter: "${{ parameters.test.virtual_perf.ctest_args }}"
-          depends_on: configure
-          installExtendedTestingTools: false
-
-  - ${{ if eq(parameters.perf_tests, 'run') }}:
-      - template: common.yml
-        parameters:
-          target: Virtual
-          env: ${{ parameters.env.Virtual }}
-          cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.Virtual.cmake_args }} -DWORKER_THREADS=2"
-          cmake_env: "${{ parameters.build.Virtual.cmake_env }}"
-          suffix: "Perf_MultiThreaded"
-          artifact_name: "Virtual_Perf_MultiThreaded"
-          ctest_filter: "-R pi_basic_mt"
-          depends_on: configure
-          installExtendedTestingTools: false
-
-  - ${{ if eq(parameters.perf_tests, 'run') }}:
-      - template: common.yml
-        parameters:
-          target: SGX
-          env: ${{ parameters.env.SGX }}
-          cmake_args: "${{ parameters.build.common.cmake_args }} ${{ parameters.build.perf.cmake_args }} ${{ parameters.build.SGX.cmake_args }} -DWORKER_THREADS=2"
-          suffix: "Perf_MultiThreaded"
-          artifact_name: "SGX_Perf_MultiThreaded"
-          ctest_filter: "-R pi_basic_mt"
-          depends_on: configure
-          installExtendedTestingTools: false
-
   - ${{ if eq(parameters.perf_tests, 'run') }}:
       - template: simulation.yml
         parameters:

.azure-pipelines-templates/test.yml

@@ -7,8 +7,6 @@ parameters:
 steps:
   - script: |
       set -ex
-      df -h /dev/shm
-      du -sh /dev/shm/*
       ASAN_SYMBOLIZER=/usr/bin/llvm-symbolizer-15
       export ASAN_SYMBOLIZER_PATH=$(realpath ${ASAN_SYMBOLIZERS})
       ./tests.sh -VV --timeout ${{ parameters.ctest_timeout }} --no-compress-output -T Test ${{ parameters.ctest_filter }}

.azure-pipelines.yml

@@ -31,15 +31,15 @@ resources:
   containers:
     - container: virtual
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: snp
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
 
 variables:
   perf_or_release: perf

.daily.yml

@@ -28,15 +28,15 @@ resources:
   containers:
     - container: virtual
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-virtual-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
 
     - container: snp
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-snp-clang15
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
 
     - container: sgx
       image: ccfmsrc.azurecr.io/ccf/ci:2024-06-26-sgx
-      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /dev/shm:/dev/shm
+      options: --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx
 
 jobs:
   - template: .azure-pipelines-templates/configure.yml

.daily_canary

@@ -3,4 +3,4 @@
  (  V  ) /  .  \  |    +---=---'
 /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
 2024__
-!..!
+!..!!

.github/workflows/build.yml

@@ -19,15 +19,15 @@ jobs:
           - name: virtual
             image: default
             nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
-            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
           - name: snp
             image: default
             nodes: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
-            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE -v /lib/modules:/lib/modules:ro
           - name: sgx
             image: sgx
             nodes: [self-hosted, 1ES.Pool=gha-sgx-ccf-sub]
-            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro -v /dev/shm:/dev/shm
+            options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --device /dev/sgx_enclave:/dev/sgx_enclave --device /dev/sgx_provision:/dev/sgx_provision -v /dev/sgx:/dev/sgx -v /lib/modules:/lib/modules:ro
     runs-on: ${{ matrix.platform.nodes }}
     container:
       image: ghcr.io/microsoft/ccf/ci/${{ matrix.platform.image }}:build-26-06-2024
@@ -68,21 +68,16 @@ jobs:
           cd build
           rm -rf /github/home/.cache
           mkdir -p /github/home/.cache
-          rm -rf /dev/shm/*
-          df -h /dev/shm
           export ASAN_SYMBOLIZER_PATH=$(realpath /usr/bin/llvm-symbolizer-15)
+          # Unit tests
+          ./tests.sh --output-on-failure -L unit -j$(nproc --all)
+          # All other acceptably fast tests, mostly end-to-end
+          ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite|unit"
+          # Partitions tests
           ./tests.sh --timeout 360 --output-on-failure -LE "benchmark|perf|protocolstest|vegeta|suite"
         shell: bash
         if: "${{ matrix.platform.name != 'snp' }}" # Needs 1ES Pool support
 
-      - name: "Partitions Test ${{ matrix.platform.name }}"
-        run: |
-          set -ex
-          cd build
-          source env/bin/activate && ctest -VV --timeout 240 -L partitions -C partitions
-        shell: bash
-        if: "${{ matrix.platform.name != 'snp' }}" # Needs 1ES Pool support
-
       - name: "Upload logs for ${{ matrix.platform.name }}"
         uses: actions/upload-artifact@v4
         with:

CHANGELOG.md

@@ -17,8 +17,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   - `::threading` is now `ccf::threading`, and `ccf/ds/thread_ids.h` has moved to `ccf/threading/thread_ids.h`
   - `::consensus` is now `ccf::consensus`
   - `::tls` is now `ccf::tls`
+  - `::http` is now `ccf::http`
   - `::nonstd` is now `ccf::nonstd`
   - `::crypto` is now `ccf::crypto`
+  - `::kv` is now `ccf::kv`
+  - `::logger` is now `ccf::logger`
+  - `::ccfapp` is now `::ccf`
 - The `programmability` sample app now demonstrates how applications can define their own extensions, creating bindings between C++ and JS state, and allowing JS endpoints to call functions implemented in C++.
 - Introduce `DynamicJSEndpointRegistry::record_action_for_audit_v1` and `DynamicJSEndpointRegistry::check_action_not_replayed_v1` to allow an application making use of the programmability feature to easily implement auditability, and protect users allowed to update the application against replay attacks (#6285).
 - Endpoints now support a `ToBackup` redirection strategy, for requests which should never be executed on a primary. These must also be read-only. These are configured similar to `ToPrimary` endpoints, with a `to_backup` object (specifying by-role or statically-addressed targets) in each node's configuration.

cmake/common.cmake

@@ -17,7 +17,7 @@ function(add_unit_test name)
   set_property(
     TEST ${name}
     APPEND
-    PROPERTY LABELS unit_test
+    PROPERTY LABELS unit
   )
 
   set_property(

doc/architecture/ledger.rst

@@ -35,21 +35,21 @@ The following table describes the structure of a serialised transaction as it is
 +          +------------------------------------------+-------------------------------------------------------------------------+
 |          | uint64_t                                 | Length of serialised public domain                                      |
 +----------+------------------------------------------+-------------------------------------------------------------------------+
-|          | :cpp:type:`kv::EntryType`                | Snapshot, or a WriteSet variant                                         |
+|          | :cpp:type:`ccf::kv::EntryType`           | Snapshot, or a WriteSet variant                                         |
 +          +------------------------------------------+-------------------------------------------------------------------------+
-|          | :cpp:type:`kv::Version`                  | Transaction version                                                     |
+|          | :cpp:type:`ccf::kv::Version`             | Transaction version                                                     |
 +          +------------------------------------------+-------------------------------------------------------------------------+
-|          | :cpp:type:`ccf::crypto::Sha256Hash`           | User-defined claims digest, when entry type is WriteSetWith.*Claims     |
+|          | :cpp:type:`ccf::crypto::Sha256Hash`      | User-defined claims digest, when entry type is WriteSetWith.*Claims     |
 +          +------------------------------------------+-------------------------------------------------------------------------+
-|          | :cpp:type:`ccf::crypto::Sha256Hash`           | Commit evidence digest, when entry type is WriteSetWithCommitEvidence.* |
+|          | :cpp:type:`ccf::crypto::Sha256Hash`      | Commit evidence digest, when entry type is WriteSetWithCommitEvidence.* |
 +          +------------------------------------------+-------------------------------------------------------------------------+
-|          | :cpp:type:`kv::Version`                  | Unused, reserved for compatibility                                      |
+|          | :cpp:type:`ccf::kv::Version`             | Unused, reserved for compatibility                                      |
 +          +------------------------------------------+-------------------------------------------------------------------------+
 |          | **Repeating [0..n]**                     | With ``n`` the number of maps in the transaction                        |
 +          +-----+------------------------------------+-------------------------------------------------------------------------+
-|          |     | std::string                        | Name of the serialised :cpp:type:`kv::Map`                              |
+|          |     | std::string                        | Name of the serialised :cpp:type:`ccf::kv::Map`                         |
 |          +-----+------------------------------------+-------------------------------------------------------------------------+
-|          |     | :cpp:type:`kv::Version`            | Read version                                                            |
+|          |     | :cpp:type:`ccf::kv::Version`       | Read version                                                            |
 |          +-----+------------------------------------+-------------------------------------------------------------------------+
 |          |     | uint64_t                           | Read count                                                              |
 |          |     +------------------------------------+-------------------------------------------------------------------------+

doc/build_apps/api.rst

@@ -10,7 +10,7 @@ A CCF application is composed of the following:
 Application Entry Point
 -----------------------
 
-.. doxygenfunction:: ccfapp::make_user_endpoints
+.. doxygenfunction:: ccf::make_user_endpoints
    :project: CCF
 
 
@@ -149,7 +149,7 @@ Indexing
 JavaScript FFI Plugins
 ----------------------
 
-.. doxygenfunction:: ccfapp::get_js_plugins
+.. doxygenfunction:: ccf::get_js_plugins
    :project: CCF
 
 HTTP Entity Tags Matching

doc/build_apps/example_cpp.rst

@@ -19,7 +19,7 @@ The Logging example application simply has:
 
 .. note::
 
-    :cpp:type:`kv::Map` tables are the only interface between CCF and the replicated application, and the sole mechanism for it to have distributed state.
+    :cpp:type:`ccf::kv::Map` tables are the only interface between CCF and the replicated application, and the sole mechanism for it to have distributed state.
 
     The Logging application keeps its state in a pair of tables, one containing private encrypted logs and the other containing public unencrypted logs. Their type is defined as:
 
@@ -46,15 +46,15 @@ The Logging example application simply has:
 Application Endpoints
 ~~~~~~~~~~~~~~~~~~~~~
 
-The implementation of :cpp:func:`ccfapp::make_user_endpoints()` should return a subclass of :cpp:class:`ccf::endpoints::EndpointRegistry`, containing the endpoints that constitute the app.
+The implementation of :cpp:func:`ccf::make_user_endpoints()` should return a subclass of :cpp:class:`ccf::endpoints::EndpointRegistry`, containing the endpoints that constitute the app.
 
 .. literalinclude:: ../../samples/apps/logging/logging.cpp
     :language: cpp
     :start-after: SNIPPET: inherit_frontend
     :lines: 1
     :dedent:
 
-The logging app defines :cpp:class:`ccfapp::LoggerHandlers`, which creates and installs handler functions or lambdas for several different HTTP endpoints. Each of these functions takes as input the details of the current request (such as the URI which was called, the query string, the request body), interacts with the KV tables using the given :cpp:class:`kv::Tx` object, and returns a result:
+The logging app defines :cpp:class:`ccf::LoggerHandlers`, which creates and installs handler functions or lambdas for several different HTTP endpoints. Each of these functions takes as input the details of the current request (such as the URI which was called, the query string, the request body), interacts with the KV tables using the given :cpp:class:`ccf::kv::Tx` object, and returns a result:
 
 .. literalinclude:: ../../samples/apps/logging/logging.cpp
     :language: cpp

doc/build_apps/kv/api.rst

@@ -3,84 +3,84 @@ Key-Value Store API
 
 This page presents the API that a CCF application must use to access and mutate the replicated key-value store.
 
-A CCF application should store its data in one or more :cpp:type:`kv::Map`. The name, type, and serialisation of these maps is under the application's control. Each invocation of an :cpp:class:`ccf::EndpointRegistry::Endpoint` is given a :cpp:class:`kv::Tx` transaction object, through which it can read and write to its :cpp:type:`kv::Map`.
+A CCF application should store its data in one or more :cpp:type:`ccf::kv::Map`. The name, type, and serialisation of these maps is under the application's control. Each invocation of an :cpp:class:`ccf::EndpointRegistry::Endpoint` is given a :cpp:class:`ccf::kv::Tx` transaction object, through which it can read and write to its :cpp:type:`ccf::kv::Map`.
 
 Map
 ---
 
-.. doxygentypedef:: kv::Version
+.. doxygentypedef:: ccf::kv::Version
    :project: CCF
 
-.. doxygenvariable:: kv::NoVersion
+.. doxygenvariable:: ccf::kv::NoVersion
    :project: CCF
 
-.. doxygenclass:: kv::TypedMap
+.. doxygenclass:: ccf::kv::TypedMap
    :project: CCF
 
-.. doxygentypedef:: kv::Map
+.. doxygentypedef:: ccf::kv::Map
    :project: CCF
 
-.. doxygenclass:: kv::TypedValue
+.. doxygenclass:: ccf::kv::TypedValue
    :project: CCF
 
-.. doxygentypedef:: kv::Value
+.. doxygentypedef:: ccf::kv::Value
    :project: CCF
 
-.. doxygenclass:: kv::TypedSet
+.. doxygenclass:: ccf::kv::TypedSet
    :project: CCF
 
-.. doxygentypedef:: kv::Set
+.. doxygentypedef:: ccf::kv::Set
    :project: CCF
 
 Transaction
 -----------
 
-.. doxygenclass:: kv::ReadOnlyTx
+.. doxygenclass:: ccf::kv::ReadOnlyTx
    :project: CCF
    :members: ro
 
-.. doxygenclass:: kv::Tx
+.. doxygenclass:: ccf::kv::Tx
    :project: CCF
    :members: rw, wo
 
 Handles
 -------
 
-.. doxygenclass:: kv::ReadableMapHandle
+.. doxygenclass:: ccf::kv::ReadableMapHandle
    :project: CCF
    :members:
 
-.. doxygenclass:: kv::WriteableMapHandle
+.. doxygenclass:: ccf::kv::WriteableMapHandle
    :project: CCF
    :members:
 
-.. doxygenclass:: kv::MapHandle
+.. doxygenclass:: ccf::kv::MapHandle
    :project: CCF
 
-.. doxygenclass:: kv::ReadableValueHandle
+.. doxygenclass:: ccf::kv::ReadableValueHandle
    :project: CCF
    :members:
 
-.. doxygenclass:: kv::WriteableValueHandle
+.. doxygenclass:: ccf::kv::WriteableValueHandle
    :project: CCF
    :members:
 
-.. doxygenclass:: kv::ValueHandle
+.. doxygenclass:: ccf::kv::ValueHandle
    :project: CCF
 
-.. doxygenclass:: kv::ReadableSetHandle
+.. doxygenclass:: ccf::kv::ReadableSetHandle
    :project: CCF
    :members:
 
-.. doxygenclass:: kv::WriteableSetHandle
+.. doxygenclass:: ccf::kv::WriteableSetHandle
    :project: CCF
    :members:
 
-.. doxygenclass:: kv::SetHandle
+.. doxygenclass:: ccf::kv::SetHandle
    :project: CCF
 
 Serialisation
 -------------
 
-.. doxygenenum:: kv::EntryType
+.. doxygenenum:: ccf::kv::EntryType
    :project: CCF