zcbor_decode: Redesign the fragment decoding API

Place fragments in the state struct so the user doesn't need to keep
track of them.

Signed-off-by: Øyvind Rønningstad <[email protected]>

README.md

@@ -103,6 +103,7 @@ Name                      | Description
 `ZCBOR_STOP_ON_ERROR`     | Enable the `stop_on_error` functionality. This makes all functions abort their execution if called when an error has already happened.
 `ZCBOR_BIG_ENDIAN`        | All decoded values are returned as big-endian. The default is little-endian.
 `ZCBOR_MAP_SMART_SEARCH`  | Applies to decoding of unordered maps. When enabled, a flag is kept for each element in an array, ensuring it is not processed twice. If disabled, a count is kept for map as a whole. Enabling increases code size and memory usage, and requires the state variable to possess the memory necessary for the flags.
+`ZCBOR_FRAGMENTS`         | Enable functions for decoding and encoding byte and text strings in fragments.
 
 
 Python script and module

include/zcbor_common.h

@@ -107,6 +107,18 @@ union {
 	bool payload_moved; /**< Is set to true while the state is stored as a backup
 	                         if @ref zcbor_update_state is called, since that function
 	                         updates the payload_end of all backed-up states. */
+	bool inside_cbor_bstr; /**< True if we are currently inside a CBOR-encoded bstr,
+	                            i.e. that as been started with zcbor_bstr_start_*(), or
+	                            `zcbor_cbor_bstr_fragments_start_*()`. */
+#ifdef ZCBOR_FRAGMENTS
+	bool inside_frag_str; /**< True if we are currently inside a fragmented (non-CBOR-encoded)
+	                           string. This is mutually exclusive with `inside_cbor_bstr`,
+	                           i.e. not set when using `zcbor_cbor_bstr_fragments_start_*()` */
+	ptrdiff_t frag_offset; /**< The offset in the current string at which this payload section starts. */
+	size_t str_total_len; /**< The total length of the string this fragment is a part of. */
+	ptrdiff_t frag_offset_cbor; /**< The offset in the current string at which this payload section starts. */
+	size_t str_total_len_cbor; /**< The total length of the string this fragment is a part of. */
+#endif
 
 /* This is the "decode state", the part of zcbor_state_t that is only used by zcbor_decode.c. */
 struct {
@@ -152,6 +164,7 @@ struct zcbor_state_constant {
 #ifdef ZCBOR_MAP_SMART_SEARCH
 	uint8_t *map_search_elem_state_end; /**< The end of the @ref map_search_elem_state buffer. */
 #endif
+	const uint8_t *curr_payload_section; /**< The currently encoded/decoded fragment. */
 };
 
 #ifdef ZCBOR_CANONICAL
@@ -281,6 +294,9 @@ do { \
 #define ZCBOR_ERR_MAP_FLAGS_NOT_AVAILABLE 20
 #define ZCBOR_ERR_INVALID_VALUE_ENCODING 21 ///! When ZCBOR_CANONICAL is defined, and the incoming data is not encoded with minimal length, or uses indefinite length array.
 #define ZCBOR_ERR_CONSTANT_STATE_MISSING 22
+#define ZCBOR_ERR_INNER_STRING_TOO_LARGE 23
+#define ZCBOR_ERR_NOT_IN_FRAGMENT 24
+#define ZCBOR_ERR_INSIDE_STRING 25
 #define ZCBOR_ERR_UNKNOWN 31
 
 /** The largest possible elem_count. */
@@ -388,17 +404,23 @@ static inline void zcbor_error(zcbor_state_t *state, int err)
 	}
 }
 
-/** Whether the current payload is exhausted. */
+/** Whether the current payload (section) is exhausted. */
 static inline bool zcbor_payload_at_end(const zcbor_state_t *state)
 {
 	return (state->payload == state->payload_end);
 }
 
-/** Update the current payload pointer (and payload_end).
+/** Introduce a new payload section.
  *
+ *  Updates the current payload pointer (and payload_end and frag_offset(_cbor)).
  *  For use when the payload is divided into multiple chunks.
  *
- *  This function also updates all backups to the new payload_end.
+ *  This function also updates all backups to the new payload_end,
+ *  and also updates the frag_offset/frag_offset_cbor of all backups.
+ *
+ *  Note that if this is called before the current payload is exhausted, the
+ *  remaining payload will be abandoned.
+ *
  *  This sets a flag so that @ref zcbor_process_backup fails if a backup is
  *  processed with the flag @ref ZCBOR_FLAG_RESTORE, but without the flag
  *  @ref ZCBOR_FLAG_KEEP_PAYLOAD since this would cause an invalid state.
@@ -411,6 +433,17 @@ static inline bool zcbor_payload_at_end(const zcbor_state_t *state)
 void zcbor_update_state(zcbor_state_t *state,
 		const uint8_t *payload, size_t payload_len);
 
+/** Get the the offset into the current string at which the current payload
+ *  pointer (state->payload) belongs. */
+size_t zcbor_current_string_offset(zcbor_state_t *state);
+
+/** Get the remaining number of bytes in the current string, calculated from
+ *  the current payload pointer (state->payload). */
+size_t zcbor_current_string_remainder(zcbor_state_t *state);
+
+/** Can be used on any fragment to tell if it is the final fragment of its string. */
+bool zcbor_is_last_fragment(const struct zcbor_string_fragment *fragment);
+
 /** Check that the provided fragments are complete and in the right order.
  *
  *  If the total length is not known, the total_len can have the value

include/zcbor_decode.h

@@ -371,7 +371,9 @@ bool zcbor_tstr_expect_term(zcbor_state_t *state, char const *str, size_t maxlen
 /** Decode and consume a bstr header.
  *
  * The rest of the string can be decoded as CBOR.
- * A state backup is created to keep track of the element count.
+ * A state backup is created to keep track of the element count and original payload_end.
+ * payload_end is set to the end of the string, so when the payload is exhausted,
+ * the string is considered fully decoded.
  * Call @ref zcbor_bstr_end_decode when done decoding the contents of the bstr.
  *
  * @param[inout] state   The current state of the decoding.
@@ -393,49 +395,45 @@ bool zcbor_bstr_start_decode(zcbor_state_t *state, struct zcbor_string *result);
 bool zcbor_bstr_end_decode(zcbor_state_t *state);
 
 
-/** Supplementary string (bstr/tstr) decoding functions for fragmented payloads: */
+#ifdef ZCBOR_FRAGMENTS
 
-/** Start decoding a bstr/tstr, even if the payload contains only part of it.
- *
- * This must be followed by a call to @ref zcbor_update_state, which can be
- * followed by a call to @ref zcbor_next_fragment. Do not call this function
- * again on subsequent fragments of the same string.
+/** Start decoding a fragmented string. I.e. a string spread over non-consecutive payload sections.
  *
- * This consumes the remaining payload as long as it belongs to the string.
+ * After calling this, you can retrieve a fragment with @ref zcbor_str_fragment_decode,
+ * then update the payload with @ref zcbor_update_state.
+ * Repeat until the string is fully decoded, then call @ref zcbor_bstr_fragments_end_decode.
  */
-bool zcbor_bstr_decode_fragment(zcbor_state_t *state, struct zcbor_string_fragment *result);
-bool zcbor_tstr_decode_fragment(zcbor_state_t *state, struct zcbor_string_fragment *result);
+bool zcbor_bstr_fragments_start_decode(zcbor_state_t *state);
+bool zcbor_tstr_fragments_start_decode(zcbor_state_t *state);
 
-/** Extract the next fragment of a string.
+/** Start decoding a fragmented CBOR-encoded bytestring.
  *
- * Use this function to extract all but the first fragment.
- */
-void zcbor_next_fragment(zcbor_state_t *state,
-	struct zcbor_string_fragment *prev_fragment,
-	struct zcbor_string_fragment *result);
-
-/** Decode and consume a bstr header, assuming the payload does not contain the whole bstr.
+ * I.e. a string spread over non-consecutive payload sections.
  *
- * The rest of the string can be decoded as CBOR.
- * A state backup is created to keep track of the element count.
- * Call @ref zcbor_update_state followed by @ref zcbor_bstr_next_fragment when
- * the current payload has been exhausted.
- * Call @ref zcbor_bstr_end_decode when done decoding the contents of the bstr.
+ * This is an alternative to zcbor_*str_fragments_start_decode() to be used if the payload
+ * contains CBOR data that will be decoded directly with other zcbor_*() functions.
+ *
+ * A state backup is created to keep track of the element count and original payload_end.
+ * After calling this, you can decode elements using other zcbor functions,
+ * then update the payload with @ref zcbor_update_state.
+ * Do not use @ref zcbor_str_fragment_decode with this function.
+ * Repeat until the string is fully decoded, then call @ref zcbor_bstr_fragments_end_decode.
+ * When the current payload section contains the end of the string,
+ * payload_end is set to the end of the string, so there is no risk of decoding past the end.
  */
-bool zcbor_bstr_start_decode_fragment(zcbor_state_t *state,
-	struct zcbor_string_fragment *result);
+bool zcbor_cbor_bstr_fragments_start_decode(zcbor_state_t *state);
 
-/** Start decoding the next fragment of a string.
+/** Retrieve a string fragment.
  *
- * Use this function to extract all but the first fragment of a CBOR-encoded
- * bstr.
+ * Consume bytes from the payload until either the end of the payload or the end of the string.
+ * Do not use this function with @ref zcbor_cbor_bstr_fragments_start_decode.
  */
-void zcbor_bstr_next_fragment(zcbor_state_t *state,
-	struct zcbor_string_fragment *prev_fragment,
-	struct zcbor_string_fragment *result);
+bool zcbor_str_fragment_decode(zcbor_state_t *state, struct zcbor_string_fragment *fragment);
+
+/** Finish decoding a fragmented string. */
+bool zcbor_str_fragments_end_decode(zcbor_state_t *state);
 
-/** Can be used on any fragment to tell if it is the final fragment of the string. */
-bool zcbor_is_last_fragment(const struct zcbor_string_fragment *fragment);
+#endif /* ZCBOR_FRAGMENTS */
 
 #ifdef __cplusplus
 }

src/zcbor_common.c

@@ -34,8 +34,7 @@ bool zcbor_new_backup(zcbor_state_t *state, size_t new_elem_count)
 	 * backup would be unused. */
 	size_t i = (state->constant_state->current_backup) - 1;
 
-	memcpy(&state->constant_state->backup_list[i], state,
-		sizeof(zcbor_state_t));
+	state->constant_state->backup_list[i] = *state;
 
 	state->elem_count = new_elem_count;
 
@@ -69,8 +68,7 @@ bool zcbor_process_backup(zcbor_state_t *state, uint32_t flags,
 				ZCBOR_ERR(ZCBOR_ERR_PAYLOAD_OUTDATED);
 			}
 		}
-		memcpy(state, &state->constant_state->backup_list[i],
-			sizeof(zcbor_state_t));
+		*state = state->constant_state->backup_list[i];
 	}
 
 	if (flags & ZCBOR_FLAG_CONSUME) {
@@ -92,17 +90,11 @@ bool zcbor_process_backup(zcbor_state_t *state, uint32_t flags,
 		state->decode_state = local_copy.decode_state;
 	}
 
-	return true;
-}
+	if (state->inside_cbor_bstr) {
 
-static void update_backups(zcbor_state_t *state, uint8_t const *new_payload_end)
-{
-	if (state->constant_state) {
-		for (unsigned int i = 0; i < state->constant_state->current_backup; i++) {
-			state->constant_state->backup_list[i].payload_end = new_payload_end;
-			state->constant_state->backup_list[i].payload_moved = true;
-		}
 	}
+
+	return true;
 }
 
 
@@ -123,6 +115,7 @@ bool zcbor_union_elem_code(zcbor_state_t *state)
 	return true;
 }
 
+
 bool zcbor_union_end_code(zcbor_state_t *state)
 {
 	if (!zcbor_process_backup(state, ZCBOR_FLAG_CONSUME, state->elem_count)) {
@@ -131,6 +124,7 @@ bool zcbor_union_end_code(zcbor_state_t *state)
 	return true;
 }
 
+
 void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
 		const uint8_t *payload, size_t payload_len, size_t elem_count,
 		uint8_t *flags, size_t flags_bytes)
@@ -147,6 +141,14 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
 	state_array[0].decode_state.map_elems_processed = 0;
 	(void)flags;
 	(void)flags_bytes;
+#endif
+	state_array[0].inside_cbor_bstr = false;
+#ifdef ZCBOR_FRAGMENTS
+	state_array[0].inside_frag_str = false;
+	state_array[0].frag_offset = 0;
+	state_array[0].str_total_len = payload_len;
+	state_array[0].frag_offset_cbor = 0;
+	state_array[0].str_total_len_cbor = payload_len;
 #endif
 	state_array[0].constant_state = NULL;
 
@@ -167,19 +169,95 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
 	state_array[0].constant_state->manually_process_elem = ZCBOR_MANUALLY_PROCESS_ELEM_DEFAULT;
 #ifdef ZCBOR_MAP_SMART_SEARCH
 	state_array[0].constant_state->map_search_elem_state_end = flags + flags_bytes;
+#endif
+#ifdef ZCBOR_FRAGMENTS
+	state_array[0].constant_state->curr_payload_section = payload;
 #endif
 	if (n_states > 2) {
 		state_array[0].constant_state->backup_list = &state_array[1];
 	}
 }
 
-void zcbor_update_state(zcbor_state_t *state,
-		const uint8_t *payload, size_t payload_len)
+
+static void update_state(zcbor_state_t *state, const uint8_t *payload, size_t payload_len)
 {
+	const uint8_t *old_payload = state->payload;
+
 	state->payload = payload;
-	state->payload_end = payload + payload_len;
+	(void)old_payload;
+#ifdef ZCBOR_FRAGMENTS
+	ptrdiff_t prev_len = old_payload - state->constant_state->curr_payload_section;
+
+	state->frag_offset += prev_len;
+	state->frag_offset_cbor += prev_len;
+	if (state->inside_cbor_bstr) {
+		state->payload_end = payload + MIN(payload_len,
+			(size_t)((ptrdiff_t)state->str_total_len_cbor - state->frag_offset_cbor));
+	} else
+#endif
+	{
+		state->payload_end = payload + payload_len;
+	}
+}
+
+
+static void update_backups(zcbor_state_t *state, const uint8_t *old_payload, size_t new_payload_len)
+{
+	for (unsigned int i = 0; i < state->constant_state->current_backup; i++) {
+		state->constant_state->backup_list[i].payload = old_payload;
+		update_state(&state->constant_state->backup_list[i], state->payload, new_payload_len);
+		state->constant_state->backup_list[i].payload_moved = true;
+	}
+}
 
-	update_backups(state, state->payload_end);
+
+void zcbor_update_state(zcbor_state_t *state, const uint8_t *payload, size_t payload_len)
+{
+	const uint8_t *old_payload = state->payload;
+	update_state(state, payload, payload_len);
+	update_backups(state, old_payload, payload_len);
+	state->constant_state->curr_payload_section = payload;
+}
+
+
+#ifdef ZCBOR_FRAGMENTS
+size_t zcbor_current_string_offset(zcbor_state_t *state)
+{
+	ptrdiff_t res;
+
+	if (state->inside_frag_str) {
+		zcbor_assert((state->payload >= state->constant_state->curr_payload_section)
+			&& (state->payload < (state->constant_state->curr_payload_section + state->str_total_len)),
+			"Payload not within fragment\n");
+		res = ((state->payload - state->constant_state->curr_payload_section)
+				+ state->frag_offset);
+	} else {
+		zcbor_assert((state->payload >= state->constant_state->curr_payload_section)
+			&& (state->payload < (state->constant_state->curr_payload_section + state->str_total_len_cbor)),
+			"Payload not within fragment\n");
+		res = ((state->payload - state->constant_state->curr_payload_section)
+				+ state->frag_offset_cbor);
+	}
+	zcbor_assert(res >= 0, "Negative offset\n");
+	return (size_t)res;
+}
+
+
+size_t zcbor_current_string_remainder(zcbor_state_t *state)
+{
+	size_t curr_offset = zcbor_current_string_offset(state);
+	if (state->inside_frag_str) {
+		return state->str_total_len - curr_offset;
+	} else {
+		return state->str_total_len_cbor - curr_offset;
+	}
+}
+#endif
+
+
+bool zcbor_is_last_fragment(const struct zcbor_string_fragment *fragment)
+{
+	return (fragment->total_len == (fragment->offset + fragment->fragment.len));
 }