Merge pull request #17909 from jackdelv/HPCC-30524

HPCC-30524 Parquet Strings cannot be converted to REAL Reviewed-By: Dan S. Camper <[email protected]> Reviewed-by: Jake Smith <[email protected]> Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Oct 24, 2023 · 2293360 · 2293360
2 parents dee5965 + b03b83d
commit 2293360
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 5 deletions.
diff --git a/plugins/parquet/parquetembed.cpp b/plugins/parquet/parquetembed.cpp
@@ -978,12 +978,44 @@ __int64 ParquetRowBuilder::getCurrIntValue(const RtlFieldInfo *field)
         {
             __int64 myint64 = 0;
             auto scalar = getCurrView(field);
-            handleDeserializeOutcome(tokenDeserializer.deserialize(scalar.data(), myint64), "signed", scalar.data());
+            std::string scalarStr(scalar.data(), scalar.size());
+            handleDeserializeOutcome(tokenDeserializer.deserialize(scalarStr.c_str(), myint64), "signed", scalarStr.c_str());
             return myint64;
         }
     }
 }
 
+double ParquetRowBuilder::getCurrRealValue(const RtlFieldInfo *field)
+{
+    switch ((*array_visitor)->type)
+    {
+        case BoolType:
+            return (*array_visitor)->bool_arr->Value(currArrayIndex());
+        case IntType:
+            return getSigned(array_visitor, currArrayIndex());
+        case UIntType:
+            return getUnsigned(array_visitor, currArrayIndex());
+        case RealType:
+            return getReal(array_visitor, currArrayIndex());
+        case DateType:
+            return (*array_visitor)->size == 32 ? (*array_visitor)->date32_arr->Value(currArrayIndex()) : (*array_visitor)->date64_arr->Value(currArrayIndex());
+        case TimestampType:
+            return (*array_visitor)->timestamp_arr->Value(currArrayIndex());
+        case TimeType:
+            return (*array_visitor)->size == 32 ? (*array_visitor)->time32_arr->Value(currArrayIndex()) : (*array_visitor)->time64_arr->Value(currArrayIndex());
+        case DurationType:
+            return (*array_visitor)->duration_arr->Value(currArrayIndex());
+        default:
+        {
+            double mydouble = 0.0;
+            auto scalar = getCurrView(field);
+            std::string scalarStr(scalar.data(), scalar.size());
+            handleDeserializeOutcome(tokenDeserializer.deserialize(scalarStr.c_str(), mydouble), "real", scalarStr.c_str());
+            return mydouble;
+        }
+    }
+}
+
 /**
  * @brief Gets a Boolean result for an ECL Row
  *
@@ -1042,10 +1074,7 @@ double ParquetRowBuilder::getRealResult(const RtlFieldInfo *field)
         return p.doubleResult;
     }
 
-    if ((*array_visitor)->type == RealType)
-        return getReal(array_visitor, currArrayIndex());
-    else
-        return getCurrIntValue(field);
+    return getCurrRealValue(field);
 }
 
 /**

diff --git a/plugins/parquet/parquetembed.hpp b/plugins/parquet/parquetembed.hpp
@@ -900,6 +900,7 @@ class ParquetRowBuilder : public CInterfaceOf<IFieldSource>
     const std::shared_ptr<arrow::Array> &getChunk(std::shared_ptr<arrow::ChunkedArray> *column);
     std::string_view getCurrView(const RtlFieldInfo *field);
     __int64 getCurrIntValue(const RtlFieldInfo *field);
+    double getCurrRealValue(const RtlFieldInfo *field);
     void nextField(const RtlFieldInfo *field);
     void nextFromStruct(const RtlFieldInfo *field);
     void xpathOrName(StringBuffer &outXPath, const RtlFieldInfo *field) const;