From 93ed118c806501d1fced0296d40b2e85ba2bf698 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 18:19:58 -0400 Subject: [PATCH 01/18] there's no reason why we should be constantly serializing and deserializing the portable version of roaring bitmap (that allows it to work in java and go) when we can just handle that case in roaringbitmap_in/roaringbitmap_out. TODO: fix up tests --- roaringbitmap.c | 128 +++++++++++++++++++++++++----------------------- 1 file changed, 67 insertions(+), 61 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 9729b76..18694f0 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -142,7 +142,7 @@ rb_from_bytea(PG_FUNCTION_ARGS) { bytea *serializedbytes = PG_GETARG_BYTEA_P(0); roaring_bitmap_t *r1; - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -261,9 +261,9 @@ roaringbitmap_in(PG_FUNCTION_ARGS) { errmsg("malformed bitmap literal"))); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -280,18 +280,23 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { roaring_uint32_iterator_t iterator; StringInfoData buf; roaring_bitmap_t *r1; - - if(rbitmap_output_format == RBITMAP_OUTPUT_BYTEA){ - return DirectFunctionCall1(byteaout, PG_GETARG_DATUM(0)); - } - + size_t expectedsize; + serializedbytes = PG_GETARG_BYTEA_P(0); - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); + if(rbitmap_output_format == RBITMAP_OUTPUT_BYTEA){ + expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_free(r1); + return DirectFunctionCall1(byteaout, PointerGetDatum(serializedbytes)); + } + initStringInfo(&buf); appendStringInfoChar(&buf, '{'); @@ -309,6 +314,7 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { appendStringInfoChar(&buf, '}'); + roaring_bitmap_free(r1); PG_RETURN_CSTRING(buf.data); } @@ -350,13 +356,13 @@ rb_or(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes2)); + r2 = roaring_bitmap_deserialize(VARDATA(serializedbytes2)); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -365,9 +371,9 @@ rb_or(PG_FUNCTION_ARGS) { } roaring_bitmap_or_inplace(r1, r2); roaring_bitmap_free(r2); - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -454,9 +460,9 @@ rb_and(PG_FUNCTION_ARGS) { errmsg("bitmap format is error"))); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r); + expectedsize = roaring_bitmap_size_in_bytes(r); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -544,9 +550,9 @@ rb_andnot(PG_FUNCTION_ARGS) { errmsg("bitmap format is error"))); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r); + expectedsize = roaring_bitmap_size_in_bytes(r); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -608,13 +614,13 @@ rb_xor(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes2)); + r2 = roaring_bitmap_deserialize(VARDATA(serializedbytes2)); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -624,9 +630,9 @@ rb_xor(PG_FUNCTION_ARGS) { roaring_bitmap_xor_inplace(r1, r2); roaring_bitmap_free(r2); - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1002,7 +1008,7 @@ rb_add(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - roaring_bitmap_t *r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + roaring_bitmap_t *r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1010,9 +1016,9 @@ rb_add(PG_FUNCTION_ARGS) { roaring_bitmap_add(r1, value); - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1030,7 +1036,7 @@ rb_remove(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - roaring_bitmap_t *r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + roaring_bitmap_t *r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1038,9 +1044,9 @@ rb_remove(PG_FUNCTION_ARGS) { roaring_bitmap_remove(r1, value); - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1212,7 +1218,7 @@ rb_fill(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1230,9 +1236,9 @@ rb_fill(PG_FUNCTION_ARGS) { roaring_bitmap_free(r2); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1261,7 +1267,7 @@ rb_clear(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1280,9 +1286,9 @@ rb_clear(PG_FUNCTION_ARGS) { roaring_bitmap_free(r2); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1310,7 +1316,7 @@ rb_flip(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1320,9 +1326,9 @@ rb_flip(PG_FUNCTION_ARGS) { roaring_bitmap_flip_inplace(r1, rangestart, rangeend); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1344,7 +1350,7 @@ rb_shiftright(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1383,9 +1389,9 @@ rb_shiftright(PG_FUNCTION_ARGS) { r1 = r2; } - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1415,12 +1421,12 @@ rb_range(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - + r2 = roaring_bitmap_create(); if (!r2) { roaring_bitmap_free(r1); @@ -1438,9 +1444,9 @@ rb_range(PG_FUNCTION_ARGS) { roaring_advance_uint32_iterator(&iterator); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r2); + expectedsize = roaring_bitmap_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r1); roaring_bitmap_free(r2); @@ -1469,7 +1475,7 @@ rb_range_cardinality(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1517,7 +1523,7 @@ rb_select(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1575,9 +1581,9 @@ rb_select(PG_FUNCTION_ARGS) { } } - expectedsize = roaring_bitmap_portable_size_in_bytes(r2); + expectedsize = roaring_bitmap_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r1); roaring_bitmap_free(r2); @@ -1609,10 +1615,10 @@ rb_build(PG_FUNCTION_ARGS) { roaring_bitmap_add(r1, da[n]); } - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1634,7 +1640,7 @@ rb_to_array(PG_FUNCTION_ARGS) uint64_t card1; uint32_t counter = 0; - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1686,7 +1692,7 @@ rb_iterate(PG_FUNCTION_ARGS) { oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - r1 = roaring_bitmap_portable_deserialize(VARDATA(data)); + r1 = roaring_bitmap_deserialize(VARDATA(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1742,7 +1748,7 @@ rb_or_trans(PG_FUNCTION_ARGS) { oldcontext = MemoryContextSwitchTo(aggctx); - r2 = roaring_bitmap_portable_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_deserialize(VARDATA(bb)); if (PG_ARGISNULL(0)) { r1 = r2; @@ -1827,14 +1833,14 @@ rb_and_trans(PG_FUNCTION_ARGS) { bb = PG_GETARG_BYTEA_P(1); oldcontext = MemoryContextSwitchTo(aggctx); - r2 = roaring_bitmap_portable_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_deserialize(VARDATA(bb)); MemoryContextSwitchTo(oldcontext); r1 = r2; } else { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); if (!roaring_bitmap_is_empty(r1)) { bb = PG_GETARG_BYTEA_P(1); - r2 = roaring_bitmap_portable_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_deserialize(VARDATA(bb)); oldcontext = MemoryContextSwitchTo(aggctx); roaring_bitmap_and_inplace(r1, r2); @@ -1916,7 +1922,7 @@ rb_xor_trans(PG_FUNCTION_ARGS) { oldcontext = MemoryContextSwitchTo(aggctx); - r2 = roaring_bitmap_portable_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_deserialize(VARDATA(bb)); if (PG_ARGISNULL(0)) { r1 = r2; @@ -2036,9 +2042,9 @@ rb_serialize(PG_FUNCTION_ARGS) { } else { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -2066,7 +2072,7 @@ rb_deserialize(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } else { serializedbytes = PG_GETARG_BYTEA_P(0); - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -2116,7 +2122,7 @@ rb_runoptimize(PG_FUNCTION_ARGS) { roaring_bitmap_t *r; size_t expectedsize; - r = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes)); + r = roaring_bitmap_deserialize(VARDATA(serializedbytes)); if (!r) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -2124,9 +2130,9 @@ rb_runoptimize(PG_FUNCTION_ARGS) { roaring_bitmap_run_optimize(r); - expectedsize = roaring_bitmap_portable_size_in_bytes(r); + expectedsize = roaring_bitmap_size_in_bytes(r); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r, VARDATA(serializedbytes)); + roaring_bitmap_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); From 72124954ad5b66121a2b616b47a0ae77194f8d66 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 18:25:35 -0400 Subject: [PATCH 02/18] serialize non-portable after deserializing from portable --- roaringbitmap.c | 118 ++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 18694f0..9ce1901 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -177,90 +177,88 @@ roaringbitmap_in(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); + } else { + /* int array input */ - roaring_bitmap_free(r1); - return dd; - } - /* else int array input */ - - /* Find the head char '{' */ - while (*ptr && isspace((unsigned char) *ptr)) + /* Find the head char '{' */ + while (*ptr && isspace((unsigned char) *ptr)) ptr++; - if (*ptr !='{') - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("malformed bitmap literal"))); - ptr++; + if (*ptr != '{') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("malformed bitmap literal"))); + ptr++; - r1 = roaring_bitmap_create(); + r1 = roaring_bitmap_create(); - while (*ptr && isspace((unsigned char) *ptr)) - ptr++; + while (*ptr && isspace((unsigned char) *ptr)) + ptr++; - if (*ptr != '}') { - while (*ptr) { - /* Parse int element */ - errno = 0; - l = strtol(ptr, &badp, 10); + if (*ptr != '}') { + while (*ptr) { + /* Parse int element */ + errno = 0; + l = strtol(ptr, &badp, 10); - /* We made no progress parsing the string, so bail out */ - if (ptr == badp){ - roaring_bitmap_free(r1); - ereport(ERROR, - (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("invalid input syntax for %s: \"%s\"", - "integer", ptr))); - } + /* We made no progress parsing the string, so bail out */ + if (ptr == badp) { + roaring_bitmap_free(r1); + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for %s: \"%s\"", + "integer", ptr))); + } - if (errno == ERANGE - || l < INT4_MIN || l > INT4_MAX - ){ + if (errno == ERANGE + || l < INT4_MIN || l > INT4_MAX + ) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg("value \"%s\" is out of range for type %s", ptr, - "integer"))); + errmsg("value \"%s\" is out of range for type %s", ptr, + "integer"))); } - /* Add int element to bitmap */ - roaring_bitmap_add(r1, l); + /* Add int element to bitmap */ + roaring_bitmap_add(r1, l); - /* Skip any trailing whitespace after the int element */ - ptr = badp; - while (*ptr && isspace((unsigned char) *ptr)) + /* Skip any trailing whitespace after the int element */ + ptr = badp; + while (*ptr && isspace((unsigned char) *ptr)) + ptr++; + + /* Find the element terminator ',' */ + if (*ptr != ',') + break; ptr++; - /* Find the element terminator ',' */ - if (*ptr != ',') - break; - ptr++; + /* Skip any trailing whitespace after the terminator */ + while (*ptr && isspace((unsigned char) *ptr)) + ptr++; + } - /* Skip any trailing whitespace after the terminator */ - while (*ptr && isspace((unsigned char) *ptr)) - ptr++; + /* Find the tail char '{' */ + if (*ptr != '}') { + roaring_bitmap_free(r1); + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("malformed bitmap literal"))); + } } - /* Find the tail char '{' */ - if (*ptr !='}'){ + /* Check if input end */ + ptr++; + while (*ptr && isspace((unsigned char) *ptr)) + ptr++; + + if (*ptr != '\0') { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("malformed bitmap literal"))); + errmsg("malformed bitmap literal"))); } } - /* Check if input end */ - ptr++; - while (*ptr && isspace((unsigned char) *ptr)) - ptr++; - - if (*ptr !='\0'){ - roaring_bitmap_free(r1); - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("malformed bitmap literal"))); - } - expectedsize = roaring_bitmap_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); From 83fd071e1eebb0ad898f9b5d98b78d4490d80fcd Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 19:10:53 -0400 Subject: [PATCH 03/18] have roaring_buffer_create use the non-portable version --- roaring_buffer_reader.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring_buffer_reader.c b/roaring_buffer_reader.c index 4456f22..ef03047 100644 --- a/roaring_buffer_reader.c +++ b/roaring_buffer_reader.c @@ -239,7 +239,7 @@ static bool rb_append_copy_range(roaring_array_t *ra, const roaring_buffer_t *rb /** - * Creates a new roaring buffer (from a partable serialized roaringbitmap buffer). + * Creates a new roaring buffer (from a serialized roaringbitmap buffer). * Returns NULL if error occurred. */ roaring_buffer_t *roaring_buffer_create(const char *buf, size_t buf_len){ @@ -612,7 +612,7 @@ roaring_bitmap_t *roaring_buffer_andnot(const roaring_buffer_t *x1, return empty_bitmap; } if (0 == length2) { - return roaring_bitmap_portable_deserialize(x1->buf); + return roaring_bitmap_deserialize(x1->buf); } roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); if(answer == NULL) From 993a5c302f286a5abfc152186af441df196bc7cf Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 19:27:26 -0400 Subject: [PATCH 04/18] set the var size --- roaringbitmap.c | 1 + 1 file changed, 1 insertion(+) diff --git a/roaringbitmap.c b/roaringbitmap.c index 9ce1901..18e046b 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -292,6 +292,7 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); + SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); return DirectFunctionCall1(byteaout, PointerGetDatum(serializedbytes)); } From d1ac89cca6a646ca6ee5b18034fee21e0f84e682 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 20:09:28 -0400 Subject: [PATCH 05/18] replace roaring_buffer with roaring_bitmap_frozen_view. looks like it was mostly copy-pasted from an old version of roaring bitmaps, anyway. --- expected/roaringbitmap.out | 2 +- roaring_buffer_reader.c | 1009 ------------------------------------ roaring_buffer_reader.h | 154 ------ roaringbitmap.c | 325 +++++------- 4 files changed, 130 insertions(+), 1360 deletions(-) delete mode 100644 roaring_buffer_reader.c delete mode 100644 roaring_buffer_reader.h diff --git a/expected/roaringbitmap.out b/expected/roaringbitmap.out index de70531..c7067a9 100644 --- a/expected/roaringbitmap.out +++ b/expected/roaringbitmap.out @@ -63,7 +63,7 @@ select '{}'::roaringbitmap; select '{ -1 , 2 , 555555 , -4 }'::roaringbitmap; roaringbitmap ------------------------------------------------------------------------------------ - \x3a300000030000000000000008000000ffff01002000000022000000240000000200237afcffffff + \x312c353a300000030000000000000008000000ffff01002000000022000000240000000200237afcffffff6a55000000000000000000000300000060 (1 row) set roaringbitmap.output_format='array'; diff --git a/roaring_buffer_reader.c b/roaring_buffer_reader.c deleted file mode 100644 index ef03047..0000000 --- a/roaring_buffer_reader.c +++ /dev/null @@ -1,1009 +0,0 @@ -/** - * For some roaringbitmap operation, when the input is serialized binary data, - * maybe only deserializing part of the data is enough. This file provides some - * functions that support direct reading of serialized binary data to improve - * performance in certain scenarios. - */ - -#include "roaring_buffer_reader.h" - -static inline int32_t keyscardsBinarySearch(const uint16_t *array, int32_t size, uint16_t ikey); -static inline int32_t keyscardsAdvanceUntil(const uint16_t *array, int32_t pos, int32_t length, uint16_t min); -static inline int32_t rb_get_size(const roaring_buffer_t *rb); -static inline uint16_t rb_get_key_at_index(const roaring_buffer_t *rb, uint16_t i); -static void *rb_get_container_at_index(const roaring_buffer_t *rb, uint16_t i, uint8_t *typecode); -static inline int32_t rb_get_index(const roaring_buffer_t *rb, uint16_t x); -static inline int32_t rb_advance_until(const roaring_buffer_t *rb, uint16_t x, int32_t pos); -static bool rb_append_copy_range(roaring_array_t *ra, const roaring_buffer_t *sa, - int32_t start_index, int32_t end_index); - -/** - * Good old binary search. - * Assumes that array is sorted, has logarithmic complexity. - * if the result is x, then: - * if ( x>0 ) you have array[x] = ikey - * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that array[-x-1]=ikey) - * keys the array sorted. - */ -static inline int32_t keyscardsBinarySearch(const uint16_t *array, int32_t size, - uint16_t ikey) { - int32_t low = 0; - int32_t high = size - 1; - while (low <= high) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = array[middleIndex << 1]; - if (middleValue < ikey) { - low = middleIndex + 1; - } else if (middleValue > ikey) { - high = middleIndex - 1; - } else { - return middleIndex; - } - } - return -(low + 1); -} - -/** - * Galloping search - * Assumes that array is sorted, has logarithmic complexity. - * if the result is x, then if x = length, you have that all values in array between pos and length - * are smaller than min. - * otherwise returns the first index x such that array[x] >= min. - */ -static inline int32_t keyscardsAdvanceUntil(const uint16_t *array, int32_t pos, - int32_t length, uint16_t min) { - int32_t lower = pos + 1; - - if ((lower >= length) || (array[lower << 1] >= min)) { - return lower; - } - - int32_t spansize = 1; - - while ((lower + spansize < length) && (array[(lower + spansize) << 1] < min)) { - spansize <<= 1; - } - int32_t upper = (lower + spansize < length) ? lower + spansize : length - 1; - - if (array[upper << 1] == min) { - return upper; - } - if (array[upper << 1] < min) { - // means - // array - // has no - // item - // >= min - // pos = array.length; - return length; - } - - // we know that the next-smallest span was too small - lower += (spansize >> 1); - - int32_t mid = 0; - while (lower + 1 != upper) { - mid = (lower + upper) >> 1; - if (array[mid << 1] == min) { - return mid; - } else if (array[mid << 1] < min) { - lower = mid; - } else { - upper = mid; - } - } - return upper; -} - -/** - * Get the number of containers - */ -static inline int32_t rb_get_size(const roaring_buffer_t *rb) { return rb->size; } - -/** - * Retrieves the key at index i - */ -static inline uint16_t rb_get_key_at_index(const roaring_buffer_t *rb, uint16_t i) { - return rb->keyscards[i * 2]; -} - -/** - * Retrieves the container at index i, filling in the typecode - * Return NULL if error occurred. - */ -static void *rb_get_container_at_index(const roaring_buffer_t *rb, uint16_t i, - uint8_t *typecode) -{ - if(i < 0 || i >= rb->size) { - fprintf(stderr, "i out of the range.\n"); - return NULL; - } - - size_t readbytes = rb->offsets[i]; - void *answer = NULL; - const char *buf = rb->buf + rb->offsets[i]; - uint32_t thiscard = rb->keyscards[2*i+1] + 1; - bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); - bool isrun = false; - if(rb->hasrun) { - if((rb->bitmapOfRunContainers[i / 8] & (1 << (i % 8))) != 0) { - isbitmap = false; - isrun = true; - } - } - if (isbitmap) { - // we check that the read is allowed - size_t containersize = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - readbytes += containersize; - if(readbytes > rb->buf_len) { - fprintf(stderr, "Running out of bytes while reading a bitset container.\n"); - return NULL; - } - - // it is now safe to read - bitset_container_t *c = bitset_container_create(); - if(c == NULL) {// memory allocation failure - fprintf(stderr, "Failed to allocate memory for a bitset container.\n"); - return NULL; - } - - bitset_container_read(thiscard, c, buf); - answer = c; - *typecode = BITSET_CONTAINER_TYPE; - } else if (isrun) { - // we check that the read is allowed - readbytes += sizeof(uint16_t); - if(readbytes > rb->buf_len) { - fprintf(stderr, "Running out of bytes while reading a run container (header).\n"); - return NULL; - } - uint16_t n_runs; - memcpy(&n_runs, buf, sizeof(uint16_t)); - size_t containersize = n_runs * sizeof(rle16_t); - readbytes += containersize; - if(readbytes > rb->buf_len) {// data is corrupted? - fprintf(stderr, "Running out of bytes while reading a run container.\n"); - return NULL; - } - // it is now safe to read - - run_container_t *c = run_container_create(); - if(c == NULL) {// memory allocation failure - fprintf(stderr, "Failed to allocate memory for a run container.\n"); - return NULL; - } - run_container_read(thiscard, c, buf); - answer = c; - *typecode = RUN_CONTAINER_TYPE; - } else { - // we check that the read is allowed - size_t containersize = thiscard * sizeof(uint16_t); - readbytes += containersize; - if(readbytes > rb->buf_len) {// data is corrupted? - fprintf(stderr, "Running out of bytes while reading an array container.\n"); - return NULL; - } - // it is now safe to read - array_container_t *c = - array_container_create_given_capacity(thiscard); - if(c == NULL) {// memory allocation failure - fprintf(stderr, "Failed to allocate memory for an array container.\n"); - return NULL; - } - array_container_read(thiscard, c, buf); - answer = c; - *typecode = ARRAY_CONTAINER_TYPE; - } - - return answer; -} - -/** - * Get the index corresponding to a 16-bit key - */ -static inline int32_t rb_get_index(const roaring_buffer_t *rb, uint16_t x){ - if ((rb->size == 0) || rb->keyscards[(rb->size - 1) * 2] == x) return rb->size - 1; - return keyscardsBinarySearch(rb->keyscards, rb->size, x); -} - -static inline int32_t rb_advance_until(const roaring_buffer_t *rb, uint16_t x, - int32_t pos) { - return keyscardsAdvanceUntil(rb->keyscards, pos, rb->size, x); -} - -/** - * Append new key-value pairs to ra, cloning values from rb at indexes - * [start_index, end_index) - * Return false if error occurred. - * -**/ -static bool rb_append_copy_range(roaring_array_t *ra, const roaring_buffer_t *rb, - int32_t start_index, int32_t end_index) { - bool ret; - ret = extend_array(ra, end_index - start_index); - if(!ret) - return false; - for (int32_t i = start_index; i < end_index; ++i) { - const int32_t pos = ra->size; - uint8_t container_type = 0; - void *c = rb_get_container_at_index(rb, i, &container_type); - if(c == NULL) - return false; - ra->keys[pos] = rb->keyscards[i * 2]; - ra->containers[pos] = c; - ra->typecodes[pos] = container_type; - ra->size++; - } - return true; -} - - -/** - * Creates a new roaring buffer (from a serialized roaringbitmap buffer). - * Returns NULL if error occurred. - */ -roaring_buffer_t *roaring_buffer_create(const char *buf, size_t buf_len){ - size_t readbytes; - const char * initbuf = buf; - - readbytes = sizeof(int32_t);// for cookie - if(readbytes > buf_len) { - fprintf(stderr, "Ran out of bytes while reading first 4 bytes.\n"); - return NULL; - } - uint32_t cookie; - memcpy(&cookie, buf, sizeof(int32_t)); - buf += sizeof(uint32_t); - if ((cookie & 0xFFFF) != SERIAL_COOKIE && - cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { - fprintf(stderr, "I failed to find one of the right cookies. Found %" PRIu32 "\n", - cookie); - return NULL; - } - int32_t size; - - if ((cookie & 0xFFFF) == SERIAL_COOKIE) - size = (cookie >> 16) + 1; - else { - readbytes += sizeof(int32_t); - if(readbytes > buf_len) { - fprintf(stderr, "Ran out of bytes while reading second part of the cookie.\n"); - return NULL; - } - memcpy(&size, buf, sizeof(int32_t)); - buf += sizeof(uint32_t); - } - if (size > (1<<16)) { - fprintf(stderr, "You cannot have so many containers, the data must be corrupted: %" PRId32 "\n", - size); - return NULL; // logically impossible - } - const char *bitmapOfRunContainers = NULL; - bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; - if (hasrun) { - int32_t s = (size + 7) / 8; - readbytes += s; - if(readbytes > buf_len) {// data is corrupted? - fprintf(stderr, "Ran out of bytes while reading run bitmap.\n"); - return NULL; - } - bitmapOfRunContainers = buf; - buf += s; - } - uint16_t *keyscards = (uint16_t *)buf; - - readbytes += size * 2 * sizeof(uint16_t); - if(readbytes > buf_len) { - fprintf(stderr, "Ran out of bytes while reading key-cardinality array.\n"); - return NULL; - } - buf += size * 2 * sizeof(uint16_t); - - /* make sure keyscards is 2 bytes aligned */ - bool keyscards_need_free = false; - if ((uintptr_t)keyscards % sizeof(uint16_t) != 0) { - uint16_t * tmpbuf = malloc(size * 2 * sizeof(uint16_t)); - if (tmpbuf == NULL) { - fprintf(stderr, "Failed to allocate memory for keyscards. Bailing out.\n"); - return NULL; - } - memcpy(tmpbuf, keyscards, size * 2 * sizeof(uint16_t)); - keyscards_need_free = true; - keyscards = tmpbuf; - } - - uint32_t *offsets = NULL; - bool offsets_need_free = false; - if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { - readbytes += size * 4; - if(readbytes > buf_len) {// data is corrupted? - fprintf(stderr, "Ran out of bytes while reading offsets.\n"); - if(keyscards_need_free) - free(keyscards); - return NULL; - } - - offsets = (uint32_t *)buf; - if ((uintptr_t)offsets % 4 != 0) { - uint32_t * tmpbuf = malloc(size * 4); - if (tmpbuf == NULL) { - fprintf(stderr, "Failed to allocate memory for offsets. Bailing out.\n"); - if(keyscards_need_free) - free(keyscards); - return NULL; - } - memcpy(tmpbuf, offsets, size * 4); - offsets_need_free = true; - offsets = tmpbuf; - } - // skipping the offsets - buf += size * 4; - } - else { - offsets = malloc(size * 4); - if (offsets == NULL) { - fprintf(stderr, "Failed to allocate memory for offsets. Bailing out.\n"); - if(keyscards_need_free) - free(keyscards); - return NULL; - } - offsets_need_free = true; - - // Reading the containers to fill offsets - for (int32_t k = 0; k < size; ++k) { - uint32_t thiscard = keyscards[2*k+1] + 1; - bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); - bool isrun = false; - if((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { - isbitmap = false; - isrun = true; - } - - offsets[k] = readbytes; - - if (isbitmap) { - size_t containersize = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - readbytes += containersize; - buf += containersize; - } else if (isrun) { - // we check that the read is allowed - readbytes += sizeof(uint16_t); - if(readbytes > buf_len) { - fprintf(stderr, "Running out of bytes while reading a run container (header).\n"); - if(keyscards_need_free) - free(keyscards); - free(offsets); - return NULL; - } - uint16_t n_runs; - memcpy(&n_runs, buf, sizeof(uint16_t)); - size_t containersize = n_runs * sizeof(rle16_t); - readbytes += containersize; - buf += containersize; - } else { - // we check that the read is allowed - size_t containersize = thiscard * sizeof(uint16_t); - readbytes += containersize; - buf += containersize; - } - } - } - - roaring_buffer_t *ans = (roaring_buffer_t *)malloc(sizeof(roaring_buffer_t)); - if (ans == NULL) { - fprintf(stderr, "Failed to allocate memory for roaring buffer. Bailing out.\n"); - if(keyscards_need_free) - free(keyscards); - if(offsets_need_free) - free(offsets); - return NULL; - } - - ans->buf = initbuf; - ans->buf_len = buf_len; - ans->size = size; - ans->keyscards = keyscards; - ans->offsets = offsets; - ans->bitmapOfRunContainers = bitmapOfRunContainers; - ans->hasrun = hasrun; - ans->keyscards_need_free = keyscards_need_free; - ans->offsets_need_free = offsets_need_free; - - return ans; -} - - -/** - * free roaring buffer - */ -void roaring_buffer_free(const roaring_buffer_t *rb) { - if(rb->keyscards_need_free) - free((void *)rb->keyscards); - if(rb->offsets_need_free) - free((void *)rb->offsets); - - free((void *)rb); -} - - -/** - * Get the cardinality of the bitmap (number of elements). - */ -uint64_t roaring_buffer_get_cardinality(const roaring_buffer_t *ra) { - uint64_t card = 0; - for (int i = 0; i < ra->size; ++i) - { - card += ra->keyscards[2*i+1] + 1; - } - return card; -} - - -/** - * Check if value x is present - * Return false if error occurred. - */ -bool roaring_buffer_contains(const roaring_buffer_t *r, - uint32_t val, - bool *result) { - bool answer; - const uint16_t hb = val >> 16; - /* - * the next function call involves a binary search and lots of branching. - */ - int32_t i = rb_get_index(r, hb); - if (i < 0){ - *result = false; - return true; - } - - uint8_t typecode; - // next call ought to be cheap - void *container = - rb_get_container_at_index(r, i, &typecode); - if(container == NULL) - { - return false; - } - - // rest might be a tad expensive, possibly involving another round of binary search - answer = container_contains(container, val & 0xFFFF, typecode); - container_free(container, typecode); - - *result = answer; - return true; -} - - -/** - * Check if all the elements of ra1 are also in ra2. - * Return false if error occurred. - */ -bool roaring_buffer_is_subset(const roaring_buffer_t *ra1, - const roaring_buffer_t *ra2, - bool *result) { - const int length1 = ra1->size, - length2 = ra2->size; - - int pos1 = 0, pos2 = 0; - - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = rb_get_key_at_index(ra1, pos1); - const uint16_t s2 = rb_get_key_at_index(ra2, pos2); - - if (s1 == s2) { - uint8_t container_type_1, container_type_2; - void *c1 = rb_get_container_at_index(ra1, pos1, - &container_type_1); - if(c1 == NULL) - { - return false; - } - void *c2 = rb_get_container_at_index(ra2, pos2, - &container_type_2); - if(c2 == NULL) - { - container_free(c1, container_type_1); - return false; - } - bool subset = - container_is_subset(c1, container_type_1, c2, container_type_2); - container_free(c1, container_type_1); - container_free(c2, container_type_2); - if (!subset){ - *result = false; - return true; - } - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - *result = false; - return true; - } else { // s1 > s2 - pos2 = rb_advance_until(ra2, s1, pos2); - } - } - if (pos1 == length1) - *result = true; - else - *result = false; - return true; -} - - -/** - * Computes the intersection between two bitmaps and returns new bitmap. The - * caller is responsible for memory management. - * Return NULL if error occurred. - */ -roaring_bitmap_t *roaring_buffer_and(const roaring_buffer_t *ra1, - const roaring_buffer_t *ra2) { - uint8_t container_result_type = 0; - const int length1 = ra1->size, - length2 = ra2->size; - uint32_t neededcap = length1 > length2 ? length2 : length1; - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); - if(answer == NULL) - return NULL; - - int pos1 = 0, pos2 = 0; - - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = rb_get_key_at_index(ra1, pos1); - const uint16_t s2 = rb_get_key_at_index(ra2, pos2); - - if (s1 == s2) { - uint8_t container_type_1, container_type_2; - void *c1 = rb_get_container_at_index(ra1, pos1, - &container_type_1); - if(c1 == NULL) - { - roaring_bitmap_free(answer); - return NULL; - } - void *c2 = rb_get_container_at_index(ra2, pos2, - &container_type_2); - if(c2 == NULL) - { - container_free(c1, container_type_1); - roaring_bitmap_free(answer); - return NULL; - } - void *c = container_and(c1, container_type_1, c2, container_type_2, - &container_result_type); - container_free(c1, container_type_1); - container_free(c2, container_type_2); - if(c == NULL) - { - roaring_bitmap_free(answer); - return NULL; - } - if (container_nonzero_cardinality(c, container_result_type)) { - ra_append(&answer->high_low_container, s1, c, - container_result_type); - } else { - container_free( - c, container_result_type); // otherwise:memory leak! - } - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = rb_advance_until(ra1, s2, pos1); - } else { // s1 > s2 - pos2 = rb_advance_until(ra2, s1, pos2); - } - } - return answer; -} - - -/** - * Computes the size of the difference (andnot) between two bitmaps. - * Return NULL if error occurred. - */ -roaring_bitmap_t *roaring_buffer_andnot(const roaring_buffer_t *x1, - const roaring_buffer_t *x2) { - bool ret; - uint8_t container_result_type = 0; - const int length1 = x1->size, - length2 = x2->size; - if (0 == length1) { - roaring_bitmap_t *empty_bitmap = roaring_bitmap_create(); - return empty_bitmap; - } - if (0 == length2) { - return roaring_bitmap_deserialize(x1->buf); - } - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); - if(answer == NULL) - return NULL; - - int pos1 = 0, pos2 = 0; - uint8_t container_type_1, container_type_2; - uint16_t s1 = 0; - uint16_t s2 = 0; - while (true) { - s1 = rb_get_key_at_index(x1, pos1); - s2 = rb_get_key_at_index(x2, pos2); - - if (s1 == s2) { - void *c1 = rb_get_container_at_index(x1, pos1, - &container_type_1); - if(c1 == NULL) - { - roaring_bitmap_free(answer); - return NULL; - } - void *c2 = rb_get_container_at_index(x2, pos2, - &container_type_2); - if(c2 == NULL) - { - container_free(c1, container_type_1); - roaring_bitmap_free(answer); - return NULL; - } - void *c = - container_andnot(c1, container_type_1, c2, container_type_2, - &container_result_type); - container_free(c1, container_type_1); - container_free(c2, container_type_2); - if(c == NULL) - { - roaring_bitmap_free(answer); - return NULL; - } - if (container_nonzero_cardinality(c, container_result_type)) { - ra_append(&answer->high_low_container, s1, c, - container_result_type); - } else { - container_free(c, container_result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - } else if (s1 < s2) { // s1 < s2 - const int next_pos1 = rb_advance_until(x1, s2, pos1); - ret = rb_append_copy_range(&answer->high_low_container, - x1, pos1, next_pos1); - if(!ret) - { - roaring_bitmap_free(answer); - return NULL; - } - // TODO : perhaps some of the copy_on_write should be based on - // answer rather than x1 (more stringent?). Many similar cases - pos1 = next_pos1; - if (pos1 == length1) break; - } else { // s1 > s2 - pos2 = rb_advance_until(x2, s1, pos2); - if (pos2 == length2) break; - } - } - if (pos2 == length2) { - ret = rb_append_copy_range(&answer->high_low_container, - x1, pos1, length1); - if(!ret) - { - roaring_bitmap_free(answer); - return NULL; - } - } - return answer; -} - - -/** - * Computes the size of the intersection between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_and_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result) { - const int length1 = x1->size, - length2 = x2->size; - uint64_t cardinality = 0; - int pos1 = 0, pos2 = 0; - - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = rb_get_key_at_index(x1, pos1); - const uint16_t s2 = rb_get_key_at_index(x2, pos2); - - if (s1 == s2) { - uint8_t container_type_1, container_type_2; - void *c1 = rb_get_container_at_index(x1, pos1, - &container_type_1); - if(c1 == NULL) - return false; - - void *c2 = rb_get_container_at_index(x2, pos2, - &container_type_2); - if(c2 == NULL) - { - container_free(c1, container_type_1); - return false; - } - cardinality += container_and_cardinality(c1, container_type_1, c2, - container_type_2); - container_free(c1, container_type_1); - container_free(c2, container_type_2); - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = rb_advance_until(x1, s2, pos1); - } else { // s1 > s2 - pos2 = rb_advance_until(x2, s1, pos2); - } - } - *result = cardinality; - return true; -} - - -/** - * Computes the size of the union between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_or_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result) { - bool ret; - uint64_t inter; - const uint64_t c1 = roaring_buffer_get_cardinality(x1); - const uint64_t c2 = roaring_buffer_get_cardinality(x2); - ret = roaring_buffer_and_cardinality(x1, x2, &inter); - if(!ret) - return false; - *result = c1 + c2 - inter; - return true; -} - - -/** - * Computes the size of the difference (andnot) between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_andnot_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result) { - bool ret; - uint64_t inter; - const uint64_t c1 = roaring_buffer_get_cardinality(x1); - ret = roaring_buffer_and_cardinality(x1, x2, &inter); - if(!ret) - return false; - - *result = c1 - inter; - return true; -} - - -/** - * Computes the size of the symmetric difference (andnot) between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_xor_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result) { - bool ret; - uint64_t inter; - const uint64_t c1 = roaring_buffer_get_cardinality(x1); - const uint64_t c2 = roaring_buffer_get_cardinality(x2); - ret = roaring_buffer_and_cardinality(x1, x2, &inter); - if(!ret) - return false; - *result = c1 + c2 - 2 * inter; - return true; -} - - -/** - * Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto - * distance, or the Jaccard similarity coefficient) - * - * The Jaccard index is undefined if both bitmaps are empty. - * Return false if error occurred. - */ -bool roaring_buffer_jaccard_index(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - double *result) { - bool ret; - uint64_t inter; - const uint64_t c1 = roaring_buffer_get_cardinality(x1); - const uint64_t c2 = roaring_buffer_get_cardinality(x2); - ret = roaring_buffer_and_cardinality(x1, x2, &inter); - if(!ret) - return false; - *result = (double)inter / (double)(c1 + c2 - inter); - return true; -} - - -/** - * Check whether two bitmaps intersect. - * Return false if error occurred. - */ -bool roaring_buffer_intersect(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - bool *result) { - const int length1 = x1->size, - length2 = x2->size; - int pos1 = 0, pos2 = 0; - - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = rb_get_key_at_index(x1, pos1); - const uint16_t s2 = rb_get_key_at_index(x2, pos2); - - if (s1 == s2) { - uint8_t container_type_1, container_type_2; - void *c1 = rb_get_container_at_index(x1, pos1, - &container_type_1); - if(c1 == NULL) - return false; - - void *c2 = rb_get_container_at_index(x2, pos2, - &container_type_2); - if(c2 == NULL) - { - container_free(c1, container_type_1); - return false; - } - - bool intersect = container_intersect(c1, container_type_1, c2, container_type_2); - container_free(c1, container_type_1); - container_free(c2, container_type_2); - - if(intersect){ - *result = true; - return true; - } - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = rb_advance_until(x1, s2, pos1); - } else { // s1 > s2 - pos2 = rb_advance_until(x2, s1, pos2); - } - } - *result = false; - return true; -} - - -/** -* Returns true if the bitmap is empty (cardinality is zero). -*/ -bool roaring_buffer_is_empty(const roaring_buffer_t *rb) { - return rb->size == 0; -} - - -/** - * Check if the two bitmaps contain the same elements. - * Return false if error occurred. - */ -bool roaring_buffer_equals(const roaring_buffer_t *rb1, - const roaring_buffer_t *rb2, - bool *result) { - if (rb1->size != rb2->size) { - *result = false; - return true; - } - for (int i = 0; i < rb1->size; ++i) { - if (rb1->keyscards[i * 2] != - rb2->keyscards[i * 2]) { - *result = false; - return true; - } - } - for (int i = 0; i < rb1->size; ++i) { - uint8_t container_type_1, container_type_2; - void *c1 = rb_get_container_at_index(rb1, i, - &container_type_1); - if(c1 == NULL) - return false; - - void *c2 = rb_get_container_at_index(rb2, i, - &container_type_2); - if(c2 == NULL) - { - container_free(c1, container_type_1); - return false; - } - - bool areequal = container_equals(c1,container_type_1, - c2,container_type_2); - container_free(c1, container_type_1); - container_free(c2, container_type_2); - - if (!areequal) { - *result = false; - return true; - } - } - *result = true; - return true; -} - - -/** -* Count the number of integers that are smaller or equal to x. -* Return false if error occurred. -*/ -bool roaring_buffer_rank(const roaring_buffer_t *rb, - uint32_t x, - uint64_t *result) { - uint32_t xhigh = x >> 16; - *result = 0; - for (int i = 0; i < rb->size; i++) { - uint32_t key = rb->keyscards[i * 2]; - if (xhigh < key) - { - return true; - } - else - { - uint8_t container_type; - void *c = rb_get_container_at_index(rb, i, - &container_type); - if(c == NULL) - return false; - - if (xhigh == key) { - *result += container_rank(c, container_type, x & 0xFFFF); - container_free(c, container_type); - return true; - } else{ - *result += container_get_cardinality(c, container_type); - container_free(c, container_type); - } - } - } - return true; -} - - -/** -* Get the smallest value in the set, or UINT32_MAX if the set is empty. -* Return false if error occurred. -*/ -bool roaring_buffer_minimum(const roaring_buffer_t *rb, - uint32_t *result) { - if (rb->size > 0) { - uint8_t typecode; - int i = 0; - uint32_t key = rb->keyscards[i * 2]; - void *container = rb_get_container_at_index(rb, i, &typecode); - if(container == NULL) - return false; - - uint32_t lowvalue = container_minimum(container, typecode); - *result = lowvalue | (key << 16); - }else { - *result = UINT32_MAX; - } - return true; -} - - -/** -* Get the greatest value in the set, or 0 if the set is empty. -* Return false if error occurred. -*/ -bool roaring_buffer_maximum(const roaring_buffer_t *rb, - uint32_t *result) { - if (rb->size > 0) { - uint8_t typecode; - int i = rb->size - 1; - uint32_t key = rb->keyscards[i * 2]; - void *container = rb_get_container_at_index(rb, i, &typecode); - if(container == NULL) - return false; - - uint32_t lowvalue = container_maximum(container, typecode); - *result = lowvalue | (key << 16); - }else { - *result = 0; - } - return true; -} \ No newline at end of file diff --git a/roaring_buffer_reader.h b/roaring_buffer_reader.h deleted file mode 100644 index 55416d0..0000000 --- a/roaring_buffer_reader.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef __ROARING_BUFFER_READER_H__ -#define __ROARING_BUFFER_READER_H__ - -#include "roaring.h" - -typedef struct roaring_buffer_s { - const char *buf; - size_t buf_len; - int32_t size; /* number of containers */ - const uint16_t *keyscards; - const uint32_t *offsets; - const char *bitmapOfRunContainers; - bool hasrun; - bool keyscards_need_free; - bool offsets_need_free; -} roaring_buffer_t; - - - -/** - * Creates a new roaring buffer (from a partable serialized roaringbitmap buffer). - * Returns NULL if error occurred. - */ -roaring_buffer_t *roaring_buffer_create(const char *buf, size_t buf_len); - -/** - * free roaring buffer - */ -void roaring_buffer_free(const roaring_buffer_t *rb); - -/** - * Get the cardinality of the bitmap (number of elements). - */ -uint64_t roaring_buffer_get_cardinality(const roaring_buffer_t *ra); - -/** - * Check if value x is present - * Return false if error occurred. - */ -bool roaring_buffer_contains(const roaring_buffer_t *r, - uint32_t val, - bool *result); - -/** - * Check if all the elements of ra1 are also in ra2. - * Return false if error occurred. - */ -bool roaring_buffer_is_subset(const roaring_buffer_t *ra1, - const roaring_buffer_t *ra2, - bool *result); - -/** - * Computes the intersection between two bitmaps and returns new bitmap. The - * caller is responsible for memory management. - * Return NULL if error occurred. - */ -roaring_bitmap_t *roaring_buffer_and(const roaring_buffer_t *ra1, - const roaring_buffer_t *ra2); - -/** - * Computes the size of the difference (andnot) between two bitmaps. - * Return NULL if error occurred. - */ -roaring_bitmap_t *roaring_buffer_andnot(const roaring_buffer_t *x1, - const roaring_buffer_t *x2); - -/** - * Computes the size of the intersection between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_and_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result); - -/** - * Computes the size of the union between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_or_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result); - -/** - * Computes the size of the difference (andnot) between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_andnot_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result); - -/** - * Computes the size of the difference (andnot) between two bitmaps. - * Return false if error occurred. - */ -bool roaring_buffer_xor_cardinality(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - uint64_t *result); - -/** - * Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto - * distance, or the Jaccard similarity coefficient) - * - * The Jaccard index is undefined if both bitmaps are empty. - * Return false if error occurred. - */ -bool roaring_buffer_jaccard_index(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - double *result); - -/** - * Check whether two bitmaps intersect. - * Return false if error occurred. - */ -bool roaring_buffer_intersect(const roaring_buffer_t *x1, - const roaring_buffer_t *x2, - bool *result); - -/** -* Returns true if the bitmap is empty (cardinality is zero). -*/ -bool roaring_buffer_is_empty(const roaring_buffer_t *rb); - - -/** - * Check if the two bitmaps contain the same elements. - * Return false if error occurred. - */ -bool roaring_buffer_equals(const roaring_buffer_t *rb1, - const roaring_buffer_t *rb2, - bool *result); - -/** -* Count the number of integers that are smaller or equal to x. -* Return false if error occurred. -*/ -bool roaring_buffer_rank(const roaring_buffer_t *rb, - uint32_t x, - uint64_t *reuslt); - -/** -* Get the smallest value in the set, or UINT32_MAX if the set is empty. -* Return false if error occurred. -*/ -bool roaring_buffer_minimum(const roaring_buffer_t *rb, - uint32_t *result); - -/** -* Get the greatest value in the set, or 0 if the set is empty. -* Return false if error occurred. -*/ -bool roaring_buffer_maximum(const roaring_buffer_t *rb, - uint32_t *result); - -#endif diff --git a/roaringbitmap.c b/roaringbitmap.c index 18e046b..e5ac60b 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -388,34 +388,30 @@ Datum rb_or_cardinality(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; uint64 card1; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_or_cardinality(r1, r2, &card1); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + card1 = roaring_bitmap_or_cardinality(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_INT64(card1); } @@ -428,31 +424,31 @@ Datum rb_and(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; roaring_bitmap_t *r; size_t expectedsize; bytea *serializedbytes; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - r = roaring_buffer_and(r1, r2); - roaring_buffer_free(r1); - roaring_buffer_free(r2); + r = roaring_bitmap_and(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); if (!r) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -477,34 +473,30 @@ Datum rb_and_cardinality(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; uint64 card1; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_and_cardinality(r1, r2, &card1); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + card1 = roaring_bitmap_and_cardinality(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_INT64(card1); } @@ -518,31 +510,31 @@ Datum rb_andnot(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; roaring_bitmap_t *r; size_t expectedsize; bytea *serializedbytes; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - r = roaring_buffer_andnot(r1, r2); - roaring_buffer_free(r1); - roaring_buffer_free(r2); + r = roaring_bitmap_andnot(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); if (!r) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -567,34 +559,30 @@ Datum rb_andnot_cardinality(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; uint64 card1; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_andnot_cardinality(r1, r2, &card1); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + card1 = roaring_bitmap_andnot_cardinality(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_INT64(card1); } @@ -647,34 +635,30 @@ Datum rb_xor_cardinality(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; uint64 card1; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_xor_cardinality(r1, r2, &card1); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + card1 = roaring_bitmap_xor_cardinality(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_INT64(card1); } @@ -687,18 +671,18 @@ Datum rb_cardinality(PG_FUNCTION_ARGS); Datum rb_cardinality(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; uint64 card; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - card = roaring_buffer_get_cardinality(r1); - roaring_buffer_free(r1); + card = roaring_bitmap_get_cardinality(r1); + roaring_bitmap_free(r1); PG_RETURN_INT64(card); } @@ -711,18 +695,18 @@ Datum rb_is_empty(PG_FUNCTION_ARGS); Datum rb_is_empty(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; bool isempty; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - isempty = roaring_buffer_is_empty(r1); - roaring_buffer_free(r1); + isempty = roaring_bitmap_is_empty(r1); + roaring_bitmap_free(r1); PG_RETURN_INT64(isempty); } @@ -735,23 +719,19 @@ Datum rb_exsit(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); uint32 value = PG_GETARG_UINT32(1); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; bool isexsit; bool ret; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - ret = roaring_buffer_contains(r1, value, &isexsit); - roaring_buffer_free(r1); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + isexsit = roaring_bitmap_contains(r1, value); + roaring_bitmap_free(r1); PG_RETURN_BOOL(isexsit); } @@ -764,34 +744,30 @@ Datum rb_equals(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; bool isequal; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_equals(r1, r2, &isequal); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + isequal = roaring_bitmap_equals(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_BOOL(isequal); } @@ -804,34 +780,30 @@ Datum rb_not_equals(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; bool isequal; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_equals(r1, r2, &isequal); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + isequal = roaring_bitmap_equals(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_BOOL(!isequal); } @@ -844,34 +816,30 @@ Datum rb_intersect(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; bool isintersect; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_intersect(r1, r2, &isintersect); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + isintersect = roaring_bitmap_intersect(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_BOOL(isintersect); } @@ -884,34 +852,30 @@ Datum rb_contains(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; bool iscontain; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_is_subset(r2, r1, &iscontain); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + iscontain = roaring_bitmap_is_subset(r2, r1); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_BOOL(iscontain); } @@ -924,34 +888,30 @@ Datum rb_containedby(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; bool iscontained; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_is_subset(r1, r2, &iscontained); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + iscontained = roaring_bitmap_is_subset(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_BOOL(iscontained); } @@ -964,34 +924,30 @@ Datum rb_jaccard_dist(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_buffer_t *r1; - roaring_buffer_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; double jaccard_dist; bool ret; - r1 = roaring_buffer_create(VARDATA(serializedbytes1), + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_buffer_create(VARDATA(serializedbytes2), + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - ret = roaring_buffer_jaccard_index(r1, r2, &jaccard_dist); - roaring_buffer_free(r1); - roaring_buffer_free(r2); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + jaccard_dist = roaring_bitmap_jaccard_index(r1, r2); + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); PG_RETURN_FLOAT8(jaccard_dist); } @@ -1059,29 +1015,25 @@ Datum rb_min(PG_FUNCTION_ARGS); Datum rb_min(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; uint32 min; bool ret; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - if(roaring_buffer_is_empty(r1)) + if(roaring_bitmap_is_empty(r1)) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); PG_RETURN_NULL(); } - ret = roaring_buffer_minimum(r1, &min); - roaring_buffer_free(r1); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + min = roaring_bitmap_minimum(r1); + roaring_bitmap_free(r1); PG_RETURN_UINT32(min); } @@ -1094,29 +1046,25 @@ Datum rb_max(PG_FUNCTION_ARGS); Datum rb_max(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; uint32 max; bool ret; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - if(roaring_buffer_is_empty(r1)) + if(roaring_bitmap_is_empty(r1)) { - roaring_buffer_free(r1); + roaring_bitmap_free(r1); PG_RETURN_NULL(); } - ret = roaring_buffer_maximum(r1, &max); - roaring_buffer_free(r1); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + max = roaring_bitmap_maximum(r1); + roaring_bitmap_free(r1); PG_RETURN_UINT32(max); } @@ -1129,23 +1077,19 @@ Datum rb_rank(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); uint32 value = PG_GETARG_UINT32(1); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; uint64 rank; bool ret; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - ret = roaring_buffer_rank(r1, value, &rank); - roaring_buffer_free(r1); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + rank = roaring_bitmap_rank(r1, value); + roaring_bitmap_free(r1); PG_RETURN_INT64((int64)rank); } @@ -1158,36 +1102,25 @@ Datum rb_index(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); uint32 value = PG_GETARG_UINT32(1); - roaring_buffer_t *r1; + const roaring_bitmap_t *r1; uint64 rank; int64 result; bool ret,isexsit; - r1 = roaring_buffer_create(VARDATA(data), + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - ret = roaring_buffer_contains(r1, value, &isexsit); - if(!ret) - { - roaring_buffer_free(r1); - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); - } + isexsit = roaring_bitmap_contains(r1, value); result = -1; if(isexsit) { - ret = roaring_buffer_rank(r1, value, &rank); - roaring_buffer_free(r1); - if(!ret) - ereport(ERROR, - (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + rank = roaring_bitmap_rank(r1, value); + roaring_bitmap_free(r1); result = (int64)rank - 1; } From d4f1f01e50dfc39ac32d1adfcecd9cb385620187 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 20:09:50 -0400 Subject: [PATCH 06/18] didn't mean to submit this out change... yet. --- expected/roaringbitmap.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expected/roaringbitmap.out b/expected/roaringbitmap.out index c7067a9..de70531 100644 --- a/expected/roaringbitmap.out +++ b/expected/roaringbitmap.out @@ -63,7 +63,7 @@ select '{}'::roaringbitmap; select '{ -1 , 2 , 555555 , -4 }'::roaringbitmap; roaringbitmap ------------------------------------------------------------------------------------ - \x312c353a300000030000000000000008000000ffff01002000000022000000240000000200237afcffffff6a55000000000000000000000300000060 + \x3a300000030000000000000008000000ffff01002000000022000000240000000200237afcffffff (1 row) set roaringbitmap.output_format='array'; From 6aa0c5f40adf5f97fef68d9b0f2aa17d36a08d36 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 20:11:35 -0400 Subject: [PATCH 07/18] remove reference to roaring buffer in header --- roaringbitmap.h | 1 - 1 file changed, 1 deletion(-) diff --git a/roaringbitmap.h b/roaringbitmap.h index d6c78bb..81e238d 100644 --- a/roaringbitmap.h +++ b/roaringbitmap.h @@ -47,6 +47,5 @@ bool ArrayContainsNulls(ArrayType *array); #define ARRISEMPTY(x) (ARRNELEMS(x) == 0) #include "roaring.c" -#include "roaring_buffer_reader.c" #endif From 8e92bc26bd33530f7e8ad9b35095a9360bc1e009 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 20:16:27 -0400 Subject: [PATCH 08/18] fix warnings for unused ret variables. --- roaringbitmap.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index e5ac60b..49f6b9f 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -391,7 +391,6 @@ rb_or_cardinality(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; uint64 card1; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -476,7 +475,6 @@ rb_and_cardinality(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; uint64 card1; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -562,7 +560,6 @@ rb_andnot_cardinality(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; uint64 card1; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -638,7 +635,6 @@ rb_xor_cardinality(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; uint64 card1; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -721,7 +717,6 @@ rb_exsit(PG_FUNCTION_ARGS) { uint32 value = PG_GETARG_UINT32(1); const roaring_bitmap_t *r1; bool isexsit; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); @@ -747,7 +742,6 @@ rb_equals(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; bool isequal; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -783,7 +777,6 @@ rb_not_equals(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; bool isequal; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -819,7 +812,6 @@ rb_intersect(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; bool isintersect; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -855,7 +847,6 @@ rb_contains(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; bool iscontain; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -891,7 +882,6 @@ rb_containedby(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; bool iscontained; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -927,7 +917,6 @@ rb_jaccard_dist(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; const roaring_bitmap_t *r2; double jaccard_dist; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); @@ -1017,7 +1006,6 @@ rb_min(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); const roaring_bitmap_t *r1; uint32 min; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); @@ -1048,7 +1036,6 @@ rb_max(PG_FUNCTION_ARGS) { bytea *data = PG_GETARG_BYTEA_P(0); const roaring_bitmap_t *r1; uint32 max; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); @@ -1079,7 +1066,6 @@ rb_rank(PG_FUNCTION_ARGS) { uint32 value = PG_GETARG_UINT32(1); const roaring_bitmap_t *r1; uint64 rank; - bool ret; r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); @@ -1105,7 +1091,7 @@ rb_index(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; uint64 rank; int64 result; - bool ret,isexsit; + bool isexsit; r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); From 075d64788bdd0d45d801bff7518a1f20238ad4e1 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 20:24:28 -0400 Subject: [PATCH 09/18] fix the first few outputs --- expected/roaringbitmap.out | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/expected/roaringbitmap.out b/expected/roaringbitmap.out index de70531..3f0e679 100644 --- a/expected/roaringbitmap.out +++ b/expected/roaringbitmap.out @@ -140,21 +140,21 @@ LINE 1: select '{2147483648}'::roaringbitmap; ^ -- Test Type cast select '{}'::roaringbitmap::bytea; - bytea --------------------- - \x3a30000000000000 + bytea +-------------- + \x0100000000 (1 row) select '{1}'::roaringbitmap::bytea; - bytea ----------------------------------------- - \x3a3000000100000000000000100000000100 + bytea +---------------------- + \x010100000001000000 (1 row) select '{1,9999}'::roaringbitmap::bytea; - bytea --------------------------------------------- - \x3a30000001000000000001001000000001000f27 + bytea +------------------------------ + \x0102000000010000000f270000 (1 row) select '{}'::roaringbitmap::bytea::roaringbitmap; From 728cc51df9c6c7ac500c8f5850eb3bdf9cbd3c9e Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 21:20:26 -0400 Subject: [PATCH 10/18] so when we're already serializing these bitmaps between operations, we might as well stick to the frozen serialization. --- roaringbitmap.c | 251 +++++++++++++++++++++++++++--------------------- 1 file changed, 143 insertions(+), 108 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 49f6b9f..e754640 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -140,9 +140,9 @@ PG_FUNCTION_INFO_V1(rb_from_bytea); Datum rb_from_bytea(PG_FUNCTION_ARGS) { bytea *serializedbytes = PG_GETARG_BYTEA_P(0); - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -259,9 +259,9 @@ roaringbitmap_in(PG_FUNCTION_ARGS) { } } - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -277,11 +277,11 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { bytea *serializedbytes; roaring_uint32_iterator_t iterator; StringInfoData buf; - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; size_t expectedsize; serializedbytes = PG_GETARG_BYTEA_P(0); - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -350,30 +350,32 @@ Datum rb_or(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_bitmap_t *r1; - roaring_bitmap_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; + roaring_bitmap_t *r3; size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_bitmap_deserialize(VARDATA(serializedbytes2)); + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); } - roaring_bitmap_or_inplace(r1, r2); + r3 = roaring_bitmap_or(r1, r2); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); + roaring_bitmap_free(r3); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -454,9 +456,9 @@ rb_and(PG_FUNCTION_ARGS) { errmsg("bitmap format is error"))); } - expectedsize = roaring_bitmap_size_in_bytes(r); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -539,9 +541,9 @@ rb_andnot(PG_FUNCTION_ARGS) { errmsg("bitmap format is error"))); } - expectedsize = roaring_bitmap_size_in_bytes(r); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -593,18 +595,19 @@ Datum rb_xor(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); bytea *serializedbytes2 = PG_GETARG_BYTEA_P(1); - roaring_bitmap_t *r1; - roaring_bitmap_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; + roaring_bitmap_t *r3; size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_bitmap_deserialize(VARDATA(serializedbytes2)); + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -612,12 +615,13 @@ rb_xor(PG_FUNCTION_ARGS) { errmsg("bitmap format is error"))); } - roaring_bitmap_xor_inplace(r1, r2); + r3 = roaring_bitmap_xor(r1, r2); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); + roaring_bitmap_free(r3); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -949,21 +953,25 @@ Datum rb_add(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); uint32 value = PG_GETARG_UINT32(1); + const roaring_bitmap_t *r1; + roaring_bitmap_t *r2; size_t expectedsize; bytea *serializedbytes; - roaring_bitmap_t *r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - roaring_bitmap_add(r1, value); + r2 = roaring_bitmap_copy(r1); + roaring_bitmap_free(r1); + roaring_bitmap_add(r2, value); - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_free(r2); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -979,19 +987,23 @@ rb_remove(PG_FUNCTION_ARGS) { uint32 value = PG_GETARG_UINT32(1); size_t expectedsize; bytea *serializedbytes; + const roaring_bitmap_t *r1; + roaring_bitmap_t *r2; - roaring_bitmap_t *r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - roaring_bitmap_remove(r1, value); + r2 = roaring_bitmap_copy(r1); + roaring_bitmap_free(r1); + roaring_bitmap_remove(r2, value); - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_free(r2); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -1123,8 +1135,9 @@ rb_fill(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); int64 rangestart = PG_GETARG_INT64(1); int64 rangeend = PG_GETARG_INT64(2); - roaring_bitmap_t *r1; - roaring_bitmap_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; + roaring_bitmap_t *r3; size_t expectedsize; bytea *serializedbytes; @@ -1136,7 +1149,7 @@ rb_fill(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1150,14 +1163,17 @@ rb_fill(PG_FUNCTION_ARGS) { (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("failed to create bitmap"))); } - roaring_bitmap_or_inplace(r1, r2); + r3 = roaring_bitmap_or(r1, r2); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); + } else { + r3 = (roaring_bitmap_t *)r1; } - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); + roaring_bitmap_free(r3); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -1172,8 +1188,9 @@ rb_clear(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); int64 rangestart = PG_GETARG_INT64(1); int64 rangeend = PG_GETARG_INT64(2); - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; roaring_bitmap_t *r2; + roaring_bitmap_t *r3; size_t expectedsize; bytea *serializedbytes; @@ -1185,7 +1202,7 @@ rb_clear(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1200,14 +1217,17 @@ rb_clear(PG_FUNCTION_ARGS) { errmsg("failed to create bitmap"))); } - roaring_bitmap_andnot_inplace(r1, r2); + r3 = roaring_bitmap_andnot(r1, r2); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); + } else { + r3 = (roaring_bitmap_t *)r1; } - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); + roaring_bitmap_free(r3); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -1222,7 +1242,8 @@ rb_flip(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); int64 rangestart = PG_GETARG_INT64(1); int64 rangeend = PG_GETARG_INT64(2); - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; + roaring_bitmap_t *r2; size_t expectedsize; bytea *serializedbytes; @@ -1234,20 +1255,23 @@ rb_flip(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); if (rangestart < rangeend) { - roaring_bitmap_flip_inplace(r1, rangestart, rangeend); + r2 = roaring_bitmap_flip(r1, rangestart, rangeend); + roaring_bitmap_free(r1); + } else { + r2 = (roaring_bitmap_t *)r1; } - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); + roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_free(r2); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -1262,13 +1286,13 @@ rb_shiftright(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); int64 distance = PG_GETARG_INT64(1); uint64 value; - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; roaring_bitmap_t *r2; roaring_uint32_iterator_t iterator; size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1307,9 +1331,9 @@ rb_shiftright(PG_FUNCTION_ARGS) { r1 = r2; } - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1325,7 +1349,7 @@ rb_range(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); int64 rangestart = PG_GETARG_INT64(1); int64 rangeend = PG_GETARG_INT64(2); - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; roaring_bitmap_t *r2; roaring_uint32_iterator_t iterator; size_t expectedsize; @@ -1339,7 +1363,7 @@ rb_range(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1362,9 +1386,9 @@ rb_range(PG_FUNCTION_ARGS) { roaring_advance_uint32_iterator(&iterator); } - expectedsize = roaring_bitmap_size_in_bytes(r2); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r1); roaring_bitmap_free(r2); @@ -1381,7 +1405,7 @@ rb_range_cardinality(PG_FUNCTION_ARGS) { bytea *serializedbytes1 = PG_GETARG_BYTEA_P(0); int64 rangestart = PG_GETARG_INT64(1); int64 rangeend = PG_GETARG_INT64(2); - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; roaring_uint32_iterator_t iterator; uint64 card1; @@ -1393,7 +1417,7 @@ rb_range_cardinality(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1427,7 +1451,7 @@ rb_select(PG_FUNCTION_ARGS) { int64 rangeend = PG_GETARG_INT64(5); int64 count = 0; int64 total_count = 0; - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; roaring_bitmap_t *r2; roaring_uint32_iterator_t iterator; size_t expectedsize; @@ -1441,7 +1465,7 @@ rb_select(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1499,9 +1523,9 @@ rb_select(PG_FUNCTION_ARGS) { } } - expectedsize = roaring_bitmap_size_in_bytes(r2); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r2, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r1); roaring_bitmap_free(r2); @@ -1533,10 +1557,10 @@ rb_build(PG_FUNCTION_ARGS) { roaring_bitmap_add(r1, da[n]); } - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -1551,14 +1575,14 @@ Datum rb_to_array(PG_FUNCTION_ARGS) { bytea *serializedbytes = PG_GETARG_BYTEA_P(0); - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; roaring_uint32_iterator_t *iterator; ArrayType *result; Datum *out_datums; uint64_t card1; uint32_t counter = 0; - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1600,7 +1624,7 @@ rb_iterate(PG_FUNCTION_ARGS) { MemoryContext oldcontext; roaring_uint32_iterator_t *fctx; bytea *data; - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; if (SRF_IS_FIRSTCALL()) { @@ -1610,7 +1634,7 @@ rb_iterate(PG_FUNCTION_ARGS) { oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - r1 = roaring_bitmap_deserialize(VARDATA(data)); + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1647,8 +1671,9 @@ rb_or_trans(PG_FUNCTION_ARGS) { MemoryContext aggctx; MemoryContext oldcontext; bytea *bb; - roaring_bitmap_t *r1; - roaring_bitmap_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; + roaring_bitmap_t *r3; // We must be called as a transition routine or we fail. if (!AggCheckCallContext(fcinfo, &aggctx)) @@ -1660,26 +1685,27 @@ rb_or_trans(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { PG_RETURN_NULL(); } - r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); + r3 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); } else { bb = PG_GETARG_BYTEA_P(1); oldcontext = MemoryContextSwitchTo(aggctx); - r2 = roaring_bitmap_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_frozen_view(VARDATA(bb), VARSIZE(bb)); if (PG_ARGISNULL(0)) { - r1 = r2; + r3 = (roaring_bitmap_t *)r2; } else { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); - roaring_bitmap_or_inplace(r1, r2); + r3 = roaring_bitmap_or(r1, r2); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); } MemoryContextSwitchTo(oldcontext); } - PG_RETURN_POINTER(r1); + PG_RETURN_POINTER(r3); } //bitmap or combine @@ -1731,8 +1757,9 @@ rb_and_trans(PG_FUNCTION_ARGS) { MemoryContext aggctx; MemoryContext oldcontext; bytea *bb; - roaring_bitmap_t *r1; - roaring_bitmap_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; + roaring_bitmap_t *r3; // We must be called as a transition routine or we fail. if (!AggCheckCallContext(fcinfo, &aggctx)) @@ -1744,32 +1771,35 @@ rb_and_trans(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { PG_RETURN_NULL(); } - r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); + r3 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); } else { if (PG_ARGISNULL(0) ) { /* postgres will crash when use PG_GETARG_BYTEA_PP here */ bb = PG_GETARG_BYTEA_P(1); oldcontext = MemoryContextSwitchTo(aggctx); - r2 = roaring_bitmap_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_frozen_view(VARDATA(bb), VARSIZE(bb)); MemoryContextSwitchTo(oldcontext); - r1 = r2; + r3 = (roaring_bitmap_t *)r2; } else { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); if (!roaring_bitmap_is_empty(r1)) { bb = PG_GETARG_BYTEA_P(1); - r2 = roaring_bitmap_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_frozen_view(VARDATA(bb), VARSIZE(bb)); oldcontext = MemoryContextSwitchTo(aggctx); - roaring_bitmap_and_inplace(r1, r2); + r3 = roaring_bitmap_and(r1, r2); MemoryContextSwitchTo(oldcontext); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); + } else { + r3 = (roaring_bitmap_t *)r1; } } } - PG_RETURN_POINTER(r1); + PG_RETURN_POINTER(r3); } //bitmap and combine @@ -1821,8 +1851,9 @@ rb_xor_trans(PG_FUNCTION_ARGS) { MemoryContext aggctx; MemoryContext oldcontext; bytea *bb; - roaring_bitmap_t *r1; - roaring_bitmap_t *r2; + const roaring_bitmap_t *r1; + const roaring_bitmap_t *r2; + roaring_bitmap_t *r3; // We must be called as a transition routine or we fail. if (!AggCheckCallContext(fcinfo, &aggctx)) @@ -1834,26 +1865,27 @@ rb_xor_trans(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { PG_RETURN_NULL(); } - r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); + r3 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); } else { bb = PG_GETARG_BYTEA_P(1); oldcontext = MemoryContextSwitchTo(aggctx); - r2 = roaring_bitmap_deserialize(VARDATA(bb)); + r2 = roaring_bitmap_frozen_view(VARDATA(bb), VARSIZE(bb)); if (PG_ARGISNULL(0)) { - r1 = r2; + r3 = (roaring_bitmap_t *)r2; } else { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); - roaring_bitmap_xor_inplace(r1, r2); + r3 = roaring_bitmap_xor(r1, r2); + roaring_bitmap_free(r1); roaring_bitmap_free(r2); } MemoryContextSwitchTo(oldcontext); } - PG_RETURN_POINTER(r1); + PG_RETURN_POINTER(r3); } //bitmap xor combine @@ -1960,9 +1992,9 @@ rb_serialize(PG_FUNCTION_ARGS) { } else { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); - expectedsize = roaring_bitmap_size_in_bytes(r1); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r1, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); @@ -1977,7 +2009,7 @@ Datum rb_deserialize(PG_FUNCTION_ARGS) { MemoryContext aggctx; bytea *serializedbytes; - roaring_bitmap_t *r1; + const roaring_bitmap_t *r1; // We must be called as a transition routine or we fail. if (!AggCheckCallContext(fcinfo, &aggctx)) @@ -1990,7 +2022,7 @@ rb_deserialize(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } else { serializedbytes = PG_GETARG_BYTEA_P(0); - r1 = roaring_bitmap_deserialize(VARDATA(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -2037,22 +2069,25 @@ Datum rb_runoptimize(PG_FUNCTION_ARGS); Datum rb_runoptimize(PG_FUNCTION_ARGS) { bytea *serializedbytes = PG_GETARG_BYTEA_P(0); - roaring_bitmap_t *r; + const roaring_bitmap_t *r1; + roaring_bitmap_t *r2; size_t expectedsize; - r = roaring_bitmap_deserialize(VARDATA(serializedbytes)); - if (!r) + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); + if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - roaring_bitmap_run_optimize(r); + r2 = roaring_bitmap_copy(r1); + roaring_bitmap_free(r1); + roaring_bitmap_run_optimize(r2); - expectedsize = roaring_bitmap_size_in_bytes(r); + expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_serialize(r, VARDATA(serializedbytes)); + roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); - roaring_bitmap_free(r); + roaring_bitmap_free(r2); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); PG_RETURN_BYTEA_P(serializedbytes); } From 714a4951cfe96a955540b9d85d958fcd5019fbd5 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 21:45:12 -0400 Subject: [PATCH 11/18] account for VARHDRSZ in the roaring_bitmap_frozen_view buffer length --- roaringbitmap.c | 84 ++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index e754640..7097963 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -142,7 +142,7 @@ rb_from_bytea(PG_FUNCTION_ARGS) { bytea *serializedbytes = PG_GETARG_BYTEA_P(0); const roaring_bitmap_t *r1; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -281,7 +281,7 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { size_t expectedsize; serializedbytes = PG_GETARG_BYTEA_P(0); - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -356,13 +356,13 @@ rb_or(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -395,14 +395,14 @@ rb_or_cardinality(PG_FUNCTION_ARGS) { uint64 card1; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -432,14 +432,14 @@ rb_and(PG_FUNCTION_ARGS) { bytea *serializedbytes; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -479,14 +479,14 @@ rb_and_cardinality(PG_FUNCTION_ARGS) { uint64 card1; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -517,14 +517,14 @@ rb_andnot(PG_FUNCTION_ARGS) { bytea *serializedbytes; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -564,14 +564,14 @@ rb_andnot_cardinality(PG_FUNCTION_ARGS) { uint64 card1; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -601,13 +601,13 @@ rb_xor(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); - r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2)); + r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -641,14 +641,14 @@ rb_xor_cardinality(PG_FUNCTION_ARGS) { uint64 card1; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -748,14 +748,14 @@ rb_equals(PG_FUNCTION_ARGS) { bool isequal; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -783,14 +783,14 @@ rb_not_equals(PG_FUNCTION_ARGS) { bool isequal; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -818,14 +818,14 @@ rb_intersect(PG_FUNCTION_ARGS) { bool isintersect; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -853,14 +853,14 @@ rb_contains(PG_FUNCTION_ARGS) { bool iscontain; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -888,14 +888,14 @@ rb_containedby(PG_FUNCTION_ARGS) { bool iscontained; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -923,14 +923,14 @@ rb_jaccard_dist(PG_FUNCTION_ARGS) { double jaccard_dist; r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), - VARSIZE(serializedbytes1)); + VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), - VARSIZE(serializedbytes2)); + VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, @@ -958,7 +958,7 @@ rb_add(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -990,7 +990,7 @@ rb_remove(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; roaring_bitmap_t *r2; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1149,7 +1149,7 @@ rb_fill(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1202,7 +1202,7 @@ rb_clear(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1255,7 +1255,7 @@ rb_flip(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1292,7 +1292,7 @@ rb_shiftright(PG_FUNCTION_ARGS) { size_t expectedsize; bytea *serializedbytes; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1363,7 +1363,7 @@ rb_range(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1417,7 +1417,7 @@ rb_range_cardinality(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1465,7 +1465,7 @@ rb_select(PG_FUNCTION_ARGS) { rangeend = MAX_BITMAP_RANGE_END; } - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes1), VARSIZE(serializedbytes1) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1582,7 +1582,7 @@ rb_to_array(PG_FUNCTION_ARGS) uint64_t card1; uint32_t counter = 0; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -2022,7 +2022,7 @@ rb_deserialize(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } else { serializedbytes = PG_GETARG_BYTEA_P(0); - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -2073,7 +2073,7 @@ rb_runoptimize(PG_FUNCTION_ARGS) { roaring_bitmap_t *r2; size_t expectedsize; - r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes)); + r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), From 6a20ed47cf33dbe19a5d4347c3ffb130ad4062d3 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 21:50:47 -0400 Subject: [PATCH 12/18] account for VARHDRSZ in the roaring_bitmap_frozen_view buffer length --- roaringbitmap.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 7097963..0100c6c 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -675,7 +675,7 @@ rb_cardinality(PG_FUNCTION_ARGS) { uint64 card; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -699,7 +699,7 @@ rb_is_empty(PG_FUNCTION_ARGS) { bool isempty; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -723,7 +723,7 @@ rb_exsit(PG_FUNCTION_ARGS) { bool isexsit; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1020,7 +1020,7 @@ rb_min(PG_FUNCTION_ARGS) { uint32 min; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1050,7 +1050,7 @@ rb_max(PG_FUNCTION_ARGS) { uint32 max; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1080,7 +1080,7 @@ rb_rank(PG_FUNCTION_ARGS) { uint64 rank; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1106,7 +1106,7 @@ rb_index(PG_FUNCTION_ARGS) { bool isexsit; r1 = roaring_bitmap_frozen_view(VARDATA(data), - VARSIZE(data)); + VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), @@ -1634,7 +1634,7 @@ rb_iterate(PG_FUNCTION_ARGS) { oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data)); + r1 = roaring_bitmap_frozen_view(VARDATA(data), VARSIZE(data) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), From 93cf8a8977bd65b860377147c002242de3f06b1c Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Tue, 10 Oct 2023 21:56:57 -0400 Subject: [PATCH 13/18] update three working tests: select '{}'::roaringbitmap::bytea; select '{1}'::roaringbitmap::bytea; select '{1,9999}'::roaringbitmap::bytea; --- expected/roaringbitmap.out | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/expected/roaringbitmap.out b/expected/roaringbitmap.out index 3f0e679..28e2b47 100644 --- a/expected/roaringbitmap.out +++ b/expected/roaringbitmap.out @@ -140,21 +140,21 @@ LINE 1: select '{2147483648}'::roaringbitmap; ^ -- Test Type cast select '{}'::roaringbitmap::bytea; - bytea --------------- - \x0100000000 + bytea +------------ + \xc6350000 (1 row) select '{1}'::roaringbitmap::bytea; - bytea ----------------------- - \x010100000001000000 + bytea +-------------------------- + \x01000000000002c6b50000 (1 row) select '{1,9999}'::roaringbitmap::bytea; bytea ------------------------------ - \x0102000000010000000f270000 + \x01000f270000010002c6b50000 (1 row) select '{}'::roaringbitmap::bytea::roaringbitmap; From c85242f93f249e2e25b04cdffa52795406e40c13 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Wed, 11 Oct 2023 07:03:52 -0400 Subject: [PATCH 14/18] documentation of roaring_bitmap_frozen_view says buffer has to be aligned by 32 bytes. --- roaringbitmap.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 0100c6c..f2e3705 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -260,7 +260,7 @@ roaringbitmap_in(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); @@ -373,7 +373,7 @@ rb_or(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); roaring_bitmap_free(r2); expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); roaring_bitmap_free(r3); @@ -457,7 +457,7 @@ rb_and(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); @@ -542,7 +542,7 @@ rb_andnot(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r, VARDATA(serializedbytes)); roaring_bitmap_free(r); @@ -619,7 +619,7 @@ rb_xor(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); roaring_bitmap_free(r2); expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); roaring_bitmap_free(r3); @@ -969,7 +969,7 @@ rb_add(PG_FUNCTION_ARGS) { roaring_bitmap_add(r2, value); expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r2); @@ -1001,7 +1001,7 @@ rb_remove(PG_FUNCTION_ARGS) { roaring_bitmap_remove(r2, value); expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r2); @@ -1171,7 +1171,7 @@ rb_fill(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); roaring_bitmap_free(r3); @@ -1225,7 +1225,7 @@ rb_clear(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r3); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r3, VARDATA(serializedbytes)); roaring_bitmap_free(r3); @@ -1269,7 +1269,7 @@ rb_flip(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r2); @@ -1332,7 +1332,7 @@ rb_shiftright(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); @@ -1387,7 +1387,7 @@ rb_range(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r1); roaring_bitmap_free(r2); @@ -1524,7 +1524,7 @@ rb_select(PG_FUNCTION_ARGS) { } expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r1); roaring_bitmap_free(r2); @@ -1559,7 +1559,7 @@ rb_build(PG_FUNCTION_ARGS) { expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); roaring_bitmap_free(r1); @@ -1993,7 +1993,7 @@ rb_serialize(PG_FUNCTION_ARGS) { r1 = (roaring_bitmap_t *) PG_GETARG_POINTER(0); expectedsize = roaring_bitmap_frozen_size_in_bytes(r1); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r1, VARDATA(serializedbytes)); SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); @@ -2084,7 +2084,7 @@ rb_runoptimize(PG_FUNCTION_ARGS) { roaring_bitmap_run_optimize(r2); expectedsize = roaring_bitmap_frozen_size_in_bytes(r2); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); + serializedbytes = (bytea *) pg_aligned_malloc(32, VARHDRSZ + expectedsize); roaring_bitmap_frozen_serialize(r2, VARDATA(serializedbytes)); roaring_bitmap_free(r2); From ded82c34d4ed5463a58e3bae9a6d165d0d3cfd97 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Wed, 11 Oct 2023 08:52:43 -0400 Subject: [PATCH 15/18] try to track down what's happening --- roaringbitmap.c | 106 ++++++++++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index f2e3705..4922329 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -146,7 +146,7 @@ rb_from_bytea(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 1"))); roaring_bitmap_free(r1); PG_RETURN_BYTEA_P(serializedbytes); @@ -176,7 +176,7 @@ roaringbitmap_in(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 2"))); } else { /* int array input */ @@ -285,7 +285,7 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 3"))); if(rbitmap_output_format == RBITMAP_OUTPUT_BYTEA){ expectedsize = roaring_bitmap_portable_size_in_bytes(r1); @@ -360,14 +360,14 @@ rb_or(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 4"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 5"))); } r3 = roaring_bitmap_or(r1, r2); roaring_bitmap_free(r1); @@ -399,7 +399,7 @@ rb_or_cardinality(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 6"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -407,7 +407,7 @@ rb_or_cardinality(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 7"))); } card1 = roaring_bitmap_or_cardinality(r1, r2); @@ -436,7 +436,7 @@ rb_and(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 8"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -444,7 +444,7 @@ rb_and(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 9"))); } r = roaring_bitmap_and(r1, r2); @@ -453,7 +453,7 @@ rb_and(PG_FUNCTION_ARGS) { if (!r) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 10"))); } expectedsize = roaring_bitmap_frozen_size_in_bytes(r); @@ -483,7 +483,7 @@ rb_and_cardinality(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 11"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -491,7 +491,7 @@ rb_and_cardinality(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 12"))); } card1 = roaring_bitmap_and_cardinality(r1, r2); @@ -521,7 +521,7 @@ rb_andnot(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 13"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -529,7 +529,7 @@ rb_andnot(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 14"))); } r = roaring_bitmap_andnot(r1, r2); @@ -538,7 +538,7 @@ rb_andnot(PG_FUNCTION_ARGS) { if (!r) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 15"))); } expectedsize = roaring_bitmap_frozen_size_in_bytes(r); @@ -568,7 +568,7 @@ rb_andnot_cardinality(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 16"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -576,7 +576,7 @@ rb_andnot_cardinality(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 17"))); } card1 = roaring_bitmap_andnot_cardinality(r1, r2); @@ -605,14 +605,14 @@ rb_xor(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 18"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); if (!r2) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 19"))); } r3 = roaring_bitmap_xor(r1, r2); @@ -645,7 +645,7 @@ rb_xor_cardinality(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 20"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -653,7 +653,7 @@ rb_xor_cardinality(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 21"))); } card1 = roaring_bitmap_xor_cardinality(r1, r2); @@ -679,7 +679,7 @@ rb_cardinality(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 22"))); card = roaring_bitmap_get_cardinality(r1); roaring_bitmap_free(r1); @@ -703,7 +703,7 @@ rb_is_empty(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 23"))); isempty = roaring_bitmap_is_empty(r1); roaring_bitmap_free(r1); @@ -727,7 +727,7 @@ rb_exsit(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 24"))); isexsit = roaring_bitmap_contains(r1, value); roaring_bitmap_free(r1); @@ -752,7 +752,7 @@ rb_equals(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 25"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -760,7 +760,7 @@ rb_equals(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 26"))); } isequal = roaring_bitmap_equals(r1, r2); @@ -787,7 +787,7 @@ rb_not_equals(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 27"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -795,7 +795,7 @@ rb_not_equals(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 28"))); } isequal = roaring_bitmap_equals(r1, r2); @@ -822,7 +822,7 @@ rb_intersect(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 29"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -830,7 +830,7 @@ rb_intersect(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 30"))); } isintersect = roaring_bitmap_intersect(r1, r2); @@ -857,7 +857,7 @@ rb_contains(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 31"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -865,7 +865,7 @@ rb_contains(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 32"))); } iscontain = roaring_bitmap_is_subset(r2, r1); @@ -892,7 +892,7 @@ rb_containedby(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 33"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -900,7 +900,7 @@ rb_containedby(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 34"))); } iscontained = roaring_bitmap_is_subset(r1, r2); @@ -927,7 +927,7 @@ rb_jaccard_dist(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 35"))); r2 = roaring_bitmap_frozen_view(VARDATA(serializedbytes2), VARSIZE(serializedbytes2) - VARHDRSZ); @@ -935,7 +935,7 @@ rb_jaccard_dist(PG_FUNCTION_ARGS) { roaring_bitmap_free(r1); ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 36"))); } jaccard_dist = roaring_bitmap_jaccard_index(r1, r2); @@ -962,7 +962,7 @@ rb_add(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 37"))); r2 = roaring_bitmap_copy(r1); roaring_bitmap_free(r1); @@ -994,7 +994,7 @@ rb_remove(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 38"))); r2 = roaring_bitmap_copy(r1); roaring_bitmap_free(r1); @@ -1024,7 +1024,7 @@ rb_min(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 39"))); if(roaring_bitmap_is_empty(r1)) { @@ -1054,7 +1054,7 @@ rb_max(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 40"))); if(roaring_bitmap_is_empty(r1)) { @@ -1084,7 +1084,7 @@ rb_rank(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 41"))); rank = roaring_bitmap_rank(r1, value); roaring_bitmap_free(r1); @@ -1110,7 +1110,7 @@ rb_index(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 42"))); isexsit = roaring_bitmap_contains(r1, value); @@ -1153,7 +1153,7 @@ rb_fill(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 43"))); if (rangestart < rangeend) { r2 = roaring_bitmap_from_range(rangestart, rangeend, 1); @@ -1206,7 +1206,7 @@ rb_clear(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 44"))); if (rangestart < rangeend) { r2 = roaring_bitmap_from_range(rangestart, rangeend, 1); @@ -1259,7 +1259,7 @@ rb_flip(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 45"))); if (rangestart < rangeend) { r2 = roaring_bitmap_flip(r1, rangestart, rangeend); @@ -1296,7 +1296,7 @@ rb_shiftright(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 46"))); if(distance != 0) { @@ -1367,7 +1367,7 @@ rb_range(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 47"))); r2 = roaring_bitmap_create(); if (!r2) { @@ -1421,7 +1421,7 @@ rb_range_cardinality(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 48"))); card1 = 0; roaring_init_iterator(r1, &iterator); @@ -1469,7 +1469,7 @@ rb_select(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 49"))); r2 = roaring_bitmap_create(); if (!r2) { @@ -1586,7 +1586,7 @@ rb_to_array(PG_FUNCTION_ARGS) if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 50"))); card1 = roaring_bitmap_get_cardinality(r1); @@ -1638,7 +1638,7 @@ rb_iterate(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 51"))); fctx = roaring_create_iterator(r1); @@ -2026,7 +2026,7 @@ rb_deserialize(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 52"))); // PostgreSQL's combine_aggregates() only init fcinfo->isnull once, // set fcinfo->isnull here to avoid bug https://github.com/ChenHuajun/pg_roaringbitmap/issues/6 fcinfo->isnull = false; @@ -2077,7 +2077,7 @@ rb_runoptimize(PG_FUNCTION_ARGS) { if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), - errmsg("bitmap format is error"))); + errmsg("bitmap format is error 53"))); r2 = roaring_bitmap_copy(r1); roaring_bitmap_free(r1); From ee84896d6495a67f6cf6e56ad5fa8869ba2a3236 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Wed, 11 Oct 2023 10:57:46 -0400 Subject: [PATCH 16/18] play around with getting rid of portable completely --- roaringbitmap.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 4922329..28319b4 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -172,11 +172,13 @@ roaringbitmap_in(PG_FUNCTION_ARGS) { dd = DirectFunctionCall1(byteain, PG_GETARG_DATUM(0)); serializedbytes = DatumGetByteaP(dd); - r1 = roaring_bitmap_portable_deserialize(VARDATA(serializedbytes)); + r1 = (roaring_bitmap_t *)roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error 2"))); + roaring_bitmap_free(r1); + PG_RETURN_BYTEA_P(serializedbytes); } else { /* int array input */ @@ -280,6 +282,10 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { const roaring_bitmap_t *r1; size_t expectedsize; + if(rbitmap_output_format == RBITMAP_OUTPUT_BYTEA){ + return DirectFunctionCall1(byteaout, PG_GETARG_DATUM(0)); + } + serializedbytes = PG_GETARG_BYTEA_P(0); r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) @@ -287,15 +293,6 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("bitmap format is error 3"))); - if(rbitmap_output_format == RBITMAP_OUTPUT_BYTEA){ - expectedsize = roaring_bitmap_portable_size_in_bytes(r1); - serializedbytes = (bytea *) palloc(VARHDRSZ + expectedsize); - roaring_bitmap_portable_serialize(r1, VARDATA(serializedbytes)); - roaring_bitmap_free(r1); - SET_VARSIZE(serializedbytes, VARHDRSZ + expectedsize); - return DirectFunctionCall1(byteaout, PointerGetDatum(serializedbytes)); - } - initStringInfo(&buf); appendStringInfoChar(&buf, '{'); From 2c37f5c08483aa2d91ac441fc66bbbe6c09c2459 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Wed, 11 Oct 2023 10:58:13 -0400 Subject: [PATCH 17/18] remove warning --- roaringbitmap.c | 1 - 1 file changed, 1 deletion(-) diff --git a/roaringbitmap.c b/roaringbitmap.c index 28319b4..3bfb07b 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -280,7 +280,6 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { roaring_uint32_iterator_t iterator; StringInfoData buf; const roaring_bitmap_t *r1; - size_t expectedsize; if(rbitmap_output_format == RBITMAP_OUTPUT_BYTEA){ return DirectFunctionCall1(byteaout, PG_GETARG_DATUM(0)); From 7ae7ceaa3206b02f1c7fb3eaef28fbaef8d58533 Mon Sep 17 00:00:00 2001 From: smmathews-bw-boston Date: Wed, 11 Oct 2023 11:13:20 -0400 Subject: [PATCH 18/18] checking.... --- roaringbitmap.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roaringbitmap.c b/roaringbitmap.c index 3bfb07b..08492c9 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -286,6 +286,10 @@ roaringbitmap_out(PG_FUNCTION_ARGS) { } serializedbytes = PG_GETARG_BYTEA_P(0); + if((uintptr_t)serializedbytes % 32 != 0) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("you're about to have trouble with 32 byte required alignment"))); r1 = roaring_bitmap_frozen_view(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); if (!r1) ereport(ERROR,