From 6ed1d5672ee1e0fe2b126277a39ab6745294a391 Mon Sep 17 00:00:00 2001 From: lanice Date: Thu, 3 Apr 2014 11:52:47 +0000 Subject: [PATCH 1/5] Add custom cereal archive for hyrise * use develop branch of cereal to support name-value pairs for loading * add Json::Value and std::optional as serializable types (load only) * add a serialization macro --- .../helper/cereal/HyriseCerealJsonArchive.h | 950 ++++++++++++++++++ src/lib/helper/cereal/cereal_type_jsonvalue.h | 42 + src/lib/helper/cereal/cereal_type_optional.h | 21 + src/lib/helper/cereal/serialize_macro.h | 64 ++ src/lib/helper/serialization.h | 11 + third_party/cereal | 2 +- 6 files changed, 1089 insertions(+), 1 deletion(-) create mode 100644 src/lib/helper/cereal/HyriseCerealJsonArchive.h create mode 100644 src/lib/helper/cereal/cereal_type_jsonvalue.h create mode 100644 src/lib/helper/cereal/cereal_type_optional.h create mode 100644 src/lib/helper/cereal/serialize_macro.h create mode 100644 src/lib/helper/serialization.h diff --git a/src/lib/helper/cereal/HyriseCerealJsonArchive.h b/src/lib/helper/cereal/HyriseCerealJsonArchive.h new file mode 100644 index 000000000..51f617543 --- /dev/null +++ b/src/lib/helper/cereal/HyriseCerealJsonArchive.h @@ -0,0 +1,950 @@ +/*! \file HyriseCerealJsonArchive.hpp + \brief JSON input and output archives */ +/* + Copyright (c) 2013, Randolph Voorhies, Shane Grant + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of cereal nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL RANDOLPH VOORHIES OR SHANE GRANT BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef CEREAL_ARCHIVES_JSON_HPP_ +#define CEREAL_ARCHIVES_JSON_HPP_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace cereal +{ + // ###################################################################### + //! An output archive designed to save data to JSON + /*! This archive uses RapidJSON to build serialie data to JSON. + + JSON archives provides a human readable output but at decreased + performance (both in time and space) compared to binary archives. + + JSON benefits greatly from name-value pairs, which if present, will + name the nodes in the output. If these are not present, each level + of the output will be given an automatically generated delimited name. + + The precision of the output archive controls the number of decimals output + for floating point numbers and should be sufficiently large (i.e. at least 20) + if there is a desire to have binary equality between the numbers output and + those read in. In general you should expect a loss of precision when going + from floating point to text and back. + + JSON archives do not output the size information for any dynamically sized structure + and instead infer it from the number of children for a node. This means that data + can be hand edited for dynamic sized structures and will still be readable. This + is accomplished through the cereal::SizeTag object, which will cause the archive + to output the data as a JSON array (e.g. marked by [] instead of {}), which indicates + that the container is variable sized and may be edited. + + \ingroup Archives */ + class JSONOutputArchive : public OutputArchive + { + enum class NodeType { StartObject, InObject, StartArray, InArray }; + + typedef rapidjson::GenericWriteStream WriteStream; + typedef rapidjson::PrettyWriter JSONWriter; + + public: + /*! @name Common Functionality + Common use cases for directly interacting with an JSONOutputArchive */ + //! @{ + + //! Construct, outputting to the provided stream + /*! @param stream The stream to output to. Can be a stringstream, a file stream, or + even cout! + @param precision The precision for floating point output */ + JSONOutputArchive(std::ostream & stream, int precision = std::numeric_limits::max_digits10) : + OutputArchive(this), + itsWriteStream(stream), + itsWriter(itsWriteStream, precision), + itsNextName(nullptr) + { + itsNameCounter.push(0); + itsNodeStack.push(NodeType::StartObject); + } + + //! Destructor, flushes the JSON + ~JSONOutputArchive() + { + itsWriter.EndObject(); + } + + //! Saves some binary data, encoded as a base64 string, with an optional name + /*! This will create a new node, optionally named, and insert a value that consists of + the data encoded as a base64 string */ + void saveBinaryValue( const void * data, size_t size, const char * name = nullptr ) + { + setNextName( name ); + writeName(); + + auto base64string = base64::encode( reinterpret_cast( data ), size ); + saveValue( base64string ); + }; + + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the JSONOutputArchive */ + //! @{ + + //! Starts a new node in the JSON output + /*! The node can optionally be given a name by calling setNextName prior + to creating the node + + Nodes only need to be started for types that are themselves objects or arrays */ + void startNode() + { + writeName(); + itsNodeStack.push(NodeType::StartObject); + itsNameCounter.push(0); + } + + //! Designates the most recently added node as finished + void finishNode() + { + // if we ended up serializing an empty object or array, writeName + // will never have been called - so start and then immediately end + // the object/array. + // + // We'll also end any object/arrays we happen to be in + switch(itsNodeStack.top()) + { + case NodeType::StartArray: + itsWriter.StartArray(); + case NodeType::InArray: + itsWriter.EndArray(); + break; + case NodeType::StartObject: + itsWriter.StartObject(); + case NodeType::InObject: + itsWriter.EndObject(); + break; + } + + itsNodeStack.pop(); + itsNameCounter.pop(); + } + + //! Sets the name for the next node created with startNode + void setNextName( const char * name ) + { + itsNextName = name; + } + + //! Saves a bool to the current node + void saveValue(bool b) { itsWriter.Bool(b); } + //! Saves an int to the current node + void saveValue(int i) { itsWriter.Int(i); } + //! Saves a uint to the current node + void saveValue(unsigned u) { itsWriter.Uint(u); } + //! Saves an int64 to the current node + void saveValue(int64_t i64) { itsWriter.Int64(i64); } + //! Saves a uint64 to the current node + void saveValue(uint64_t u64) { itsWriter.Uint64(u64); } + //! Saves a double to the current node + void saveValue(double d) { itsWriter.Double(d); } + //! Saves a string to the current node + void saveValue(std::string const & s) { itsWriter.String(s.c_str(), static_cast( s.size() )); } + //! Saves a const char * to the current node + void saveValue(char const * s) { itsWriter.String(s); } + +#ifdef _MSC_VER + // Visual Studio has problems disambiguating the above for unsigned long, so we provide an explicit + // overload for long and serialize it as its size necessitates + // + // When loading we don't need to do this specialization since we catch the types with + // templates according to their size + + //! 32 bit long saving to current node + template inline + typename std::enable_if::type + saveLong(T lu){ saveValue( static_cast( lu ) ); } + + //! non 32 bit long saving to current node + template inline + typename std::enable_if::type + saveLong(T lu){ saveValue( static_cast( lu ) ); } + + //! MSVC only long overload to current node + void saveValue( unsigned long lu ){ saveLong( lu ); }; +#endif + + //! Save exotic arithmetic types as binary to current node + template inline + typename std::enable_if::value && + (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), void>::type + saveValue(T const & t) + { + auto base64string = base64::encode( reinterpret_cast( &t ), sizeof(T) ); + saveValue( base64string ); + } + + //! Write the name of the upcoming node and prepare object/array state + /*! Since writeName is called for every value that is output, regardless of + whether it has a name or not, it is the place where we will do a deferred + check of our node state and decide whether we are in an array or an object. + + The general workflow of saving to the JSON archive is: + + 1. (optional) Set the name for the next node to be created, usually done by an NVP + 2. Start the node + 3. (if there is data to save) Write the name of the node (this function) + 4. (if there is data to save) Save the data (with saveValue) + 5. Finish the node + */ + void writeName() + { + NodeType const & nodeType = itsNodeStack.top(); + + // Start up either an object or an array, depending on state + if(nodeType == NodeType::StartArray) + { + itsWriter.StartArray(); + itsNodeStack.top() = NodeType::InArray; + } + else if(nodeType == NodeType::StartObject) + { + itsNodeStack.top() = NodeType::InObject; + itsWriter.StartObject(); + } + + // Array types do not output names + if(nodeType == NodeType::InArray) return; + + if(itsNextName == nullptr) + { + std::string name = "value" + std::to_string( itsNameCounter.top()++ ) + "\0"; + saveValue(name); + } + else + { + saveValue(itsNextName); + itsNextName = nullptr; + } + } + + //! Designates that the current node should be output as an array, not an object + void makeArray() + { + itsNodeStack.top() = NodeType::StartArray; + } + + //! @} + private: + WriteStream itsWriteStream; //!< Rapidjson write stream + JSONWriter itsWriter; //!< Rapidjson writer + char const * itsNextName; //!< The next name + std::stack itsNameCounter; //!< Counter for creating unique names for unnamed nodes + std::stack itsNodeStack; + }; // JSONOutputArchive + + // ###################################################################### + //! An input archive designed to load data from JSON + /*! This archive uses RapidJSON to read in a JSON archive. + + Input JSON should have been produced by the JSONOutputArchive. Data can + only be added to dynamically sized containers (marked by JSON arrays) - + the input archive will determine their size by looking at the number of child nodes. + Only JSON originating from a JSONOutputArchive is officially supported, but data + from other sources may work if properly formatted. + + The JSONInputArchive does not require that nodes are loaded in the same + order they were saved by JSONOutputArchive. Using name value pairs (NVPs), + it is possible to load in an out of order fashion or otherwise skip/select + specific nodes to load. + + The default behavior of the input archive is to read sequentially starting + with the first node and exploring its children. When a given NVP does + not match the read in name for a node, the archive will search for that + node at the current level and load it if it exists. After loading an out of + order node, the archive will then proceed back to loading sequentially from + its new position. + + Consider this simple example where loading of some data is skipped: + + @code{cpp} + // imagine the input file has someData(1-9) saved in order at the top level node + ar( someData1, someData2, someData3 ); // XML loads in the order it sees in the file + ar( cereal::make_nvp( "hello", someData6 ) ); // NVP given does not + // match expected NVP name, so we search + // for the given NVP and load that value + ar( someData7, someData8, someData9 ); // with no NVP given, loading resumes at its + // current location, proceeding sequentially + @endcode + + \ingroup Archives */ + class JSONInputArchive : public InputArchive + { + private: + typedef rapidjson::GenericReadStream ReadStream; + typedef rapidjson::GenericValue> JSONValue; + typedef JSONValue::ConstMemberIterator MemberIterator; + typedef JSONValue::ConstValueIterator ValueIterator; + typedef rapidjson::Document::GenericValue GenericValue; + + public: + /*! @name Common Functionality + Common use cases for directly interacting with an JSONInputArchive */ + //! @{ + + //! Construct, reading from the provided stream + /*! @param stream The stream to read from */ + JSONInputArchive(std::istream & stream) : + InputArchive(this), + itsNextName( nullptr ), + itsReadStream(stream) + { + itsDocument.ParseStream<0>(itsReadStream); + itsIteratorStack.emplace_back(itsDocument.MemberBegin(), itsDocument.MemberEnd()); + } + + //! Loads some binary data, encoded as a base64 string + /*! This will automatically start and finish a node to load the data, and can be called directly by + users. + + Note that this follows the same ordering rules specified in the class description in regards + to loading in/out of order */ + + void loadBinaryValue( void * data, size_t size, const char * name = nullptr ) + { + itsNextName = name; + + std::string encoded; + loadValue( encoded ); + auto decoded = base64::decode( encoded ); + + if( size != decoded.size() ) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy( data, decoded.data(), decoded.size() ); + itsNextName = nullptr; + }; + + private: + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the JSONInputArchive */ + //! @{ + + //! An internal iterator that handles both array and object types + /*! This class is a variant and holds both types of iterators that + rapidJSON supports - one for arrays and one for objects. */ + class Iterator + { + public: + Iterator() : itsIndex( 0 ), itsType(Null) {} + + Iterator(MemberIterator begin, MemberIterator end) : + itsMemberItBegin(begin), itsMemberItEnd(end), itsIndex(0), itsType(Member) + { } + + Iterator(ValueIterator begin, ValueIterator end) : + itsValueItBegin(begin), itsValueItEnd(end), itsIndex(0), itsType(Value) + { } + + //! Advance to the next node + Iterator & operator++() + { + ++itsIndex; + return *this; + } + + //! Get the value of the current node + GenericValue const & value() + { + switch(itsType) + { + case Value : return itsValueItBegin[itsIndex]; + case Member: return itsMemberItBegin[itsIndex].value; + default: throw cereal::Exception("Invalid Iterator Type!"); + } + } + + //! Get the name of the current node, or nullptr if it has no name + const char * name() const + { + if( itsType == Member && (itsMemberItBegin + itsIndex) != itsMemberItEnd ) + return itsMemberItBegin[itsIndex].name.GetString(); + else + return nullptr; + } + + //! Adjust our position such that we are at the node with the given name + /*! @throws Exception if no such named node exists */ + inline void search( const char * searchName )//, GenericValue const & parent ) + { + size_t index = 0; + for( auto it = itsMemberItBegin; it != itsMemberItEnd; ++it, ++index ) + { + if( std::strcmp( searchName, it->name.GetString() ) == 0 ) + { + itsIndex = index; + return; + } + } + + if( _optional ) + { + itsIndex = -1; + return; + } + + throw std::runtime_error("JSON Parsing failed - provided NVP not found"); + } + + bool found() + { + return itsIndex != size_t(-1); + } + + void setOptional(bool opt) + { + _optional = opt; + } + + private: + MemberIterator itsMemberItBegin, itsMemberItEnd; //!< The member iterator (object) + ValueIterator itsValueItBegin, itsValueItEnd; //!< The value iterator (array) + public: + size_t itsIndex; //!< The current index of this iterator + private: + enum Type {Value, Member, Null} itsType; //!< Whether this holds values (array) or members (objects) or nothing + bool _optional = false; + }; + + //! Searches for the expectedName node if it doesn't match the actualName + /*! This needs to be called before every load or node start occurs. This function will + check to see if an NVP has been provided (with setNextName) and if so, see if that name matches the actual + next name given. If the names do not match, it will search in the current level of the JSON for that name. + If the name is not found, an exception will be thrown. + + Resets the NVP name after called. + + @throws Exception if an expectedName is given and not found */ + inline void search() + { + // The name an NVP provided with setNextName() + if( itsNextName ) + { + // The actual name of the current node + auto const actualName = itsIteratorStack.back().name(); + + // Do a search if we don't see a name coming up, or if the names don't match + if( !actualName || std::strcmp( itsNextName, actualName ) != 0 ) + itsIteratorStack.back().search( itsNextName ); + } + + itsNextName = nullptr; + } + + public: + //! Starts a new node, going into its proper iterator + /*! This places an iterator for the next node to be parsed onto the iterator stack. If the next + node is an array, this will be a value iterator, otherwise it will be a member iterator. + + By default our strategy is to start with the document root node and then recursively iterate through + all children in the order they show up in the document. + We don't need to know NVPs to do this; we'll just blindly load in the order things appear in. + + If we were given an NVP, we will search for it if it does not match our the name of the next node + that would normally be loaded. This functionality is provided by search(). */ + void startNode() + { + search(); + + if(itsIteratorStack.back().value().IsArray()) + itsIteratorStack.emplace_back(itsIteratorStack.back().value().Begin(), itsIteratorStack.back().value().End()); + else + itsIteratorStack.emplace_back(itsIteratorStack.back().value().MemberBegin(), itsIteratorStack.back().value().MemberEnd()); + } + + //! Finishes the most recently started node + void finishNode() + { + itsIteratorStack.pop_back(); + ++itsIteratorStack.back(); + } + + //! Sets the name for the next node created with startNode + void setNextName( const char * name ) + { + itsNextName = name; + } + + //! Loads a value from the current node - small signed overload + template inline + typename std::enable_if::value && sizeof(T) < sizeof(int64_t), void>::type + loadValue(T & val) + { + search(); + + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetInt(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + + //! Loads a value from the current node - small unsigned overload + template inline + typename std::enable_if<(std::is_unsigned::value && sizeof(T) < sizeof(uint64_t)) && + !std::is_same::value, void>::type + loadValue(T & val) + { + search(); + + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetUint(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + + //! Loads a value from the current node - bool overload + void loadValue(bool & val) + { + search(); + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetBool(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + //! Loads a value from the current node - int64 overload + void loadValue(int64_t & val) + { + search(); + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetInt64(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + //! Loads a value from the current node - uint64 overload + void loadValue(uint64_t & val) + { + search(); + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetUint64(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + //! Loads a value from the current node - float overload + void loadValue(float & val) + { + search(); + if ( itsIteratorStack.back().found()) + { + val = static_cast(itsIteratorStack.back().value().GetDouble()); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + //! Loads a value from the current node - double overload + void loadValue(double & val) + { + search(); + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetDouble(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + //! Loads a value from the current node - string overload + void loadValue(std::string & val) + { + search(); + if ( itsIteratorStack.back().found()) + { + val = itsIteratorStack.back().value().GetString(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); + } + //! Loads a value from the current node - long double and long long overloads + /*! These data types will automatically be encoded as base64 strings */ + template inline + typename std::enable_if::value && + (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), void>::type + loadValue(T & val) + { + std::string encoded; + loadValue( encoded ); + auto decoded = base64::decode( encoded ); + + if( sizeof(T) != decoded.size() ) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy( &val, decoded.data(), decoded.size() ); + } + + //! Loads the size for a SizeTag + void loadSize(size_type & size) + { + size = (itsIteratorStack.rbegin() + 1)->value().Size(); + } + + int valueType() + { + return (itsIteratorStack.rbegin() + 1)->value().GetType(); + } + + std::string parseObjectToString() + { + rapidjson::StringBuffer sb; + rapidjson::Writer writer( sb ); + + itsIteratorStack.back().value().Accept( writer ); + + ++itsIteratorStack.back(); + return sb.GetString(); + } + + //! @} + + private: + const char * itsNextName; //!< Next name set by NVP + ReadStream itsReadStream; //!< Rapidjson write stream + std::vector itsIteratorStack; //!< 'Stack' of rapidJSON iterators + rapidjson::Document itsDocument; //!< Rapidjson document + + bool _foundOptionalValue; + + public: + unsigned int openJsonNodes = 0; + + bool foundOptionalValue() + { + return _foundOptionalValue; + } + + void setOptional(bool opt) + { + itsIteratorStack.back().setOptional(opt); + } + }; + + // ###################################################################### + // JSONArchive prologue and epilogue functions + // ###################################################################### + + // ###################################################################### + //! Prologue for NVPs for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void prologue( JSONOutputArchive &, NameValuePair const & ) + { } + + //! Prologue for NVPs for JSON archives + template inline + void prologue( JSONInputArchive &, NameValuePair const & ) + { } + + // ###################################################################### + //! Epilogue for NVPs for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( JSONOutputArchive &, NameValuePair const & ) + { } + + //! Epilogue for NVPs for JSON archives + /*! NVPs do not start or finish nodes - they just set up the names */ + template inline + void epilogue( JSONInputArchive &, NameValuePair const & ) + { } + + // ###################################################################### + //! Prologue for SizeTags for JSON archives + /*! SizeTags are strictly ignored for JSON, they just indicate + that the current node should be made into an array */ + template inline + void prologue( JSONOutputArchive & ar, SizeTag const & ) + { + ar.makeArray(); + } + + //! Prologue for SizeTags for JSON archives + template inline + void prologue( JSONInputArchive &, SizeTag const & ) + { } + + // ###################################################################### + //! Epilogue for SizeTags for JSON archives + /*! SizeTags are strictly ignored for JSON */ + template inline + void epilogue( JSONOutputArchive &, SizeTag const & ) + { } + + //! Epilogue for SizeTags for JSON archives + template inline + void epilogue( JSONInputArchive &, SizeTag const & ) + { } + + // ###################################################################### + //! Prologue for Json::Value for JSON archives + template inline + typename std::enable_if::value, void>::type + prologue( JSONInputArchive & ar, T const & ) + { + ++ar.openJsonNodes; + ar.startNode(); + } + + // ###################################################################### + //! Epilogue for Json::Value for JSON archives + template inline + typename std::enable_if::value, void>::type + epilogue( JSONInputArchive & ar, T const & ) + { + while (ar.openJsonNodes > 0) + { + ar.finishNode(); + --ar.openJsonNodes; + } + } + + // ###################################################################### + //! Prologue for all other types for JSON archives + /*! Starts a new node, named either automatically or by some NVP, + that may be given data by the type about to be archived */ + template inline + typename std::enable_if::value, void>::type + prologue( JSONOutputArchive & ar, T const & ) + { + ar.startNode(); + } + + //! Prologue for all other types for JSON archives + template inline + typename std::enable_if::value && !std::is_same::value, void>::type + prologue( JSONInputArchive & ar, T const & ) + { + ar.startNode(); + } + + // ###################################################################### + //! Epilogue for all other types other for JSON archives + /*! Finishes the node created in the prologue */ + template inline + typename std::enable_if::value, void>::type + epilogue( JSONOutputArchive & ar, T const & ) + { + ar.finishNode(); + } + + //! Epilogue for all other types other for JSON archives + template inline + typename std::enable_if::value && !std::is_same::value, void>::type + epilogue( JSONInputArchive & ar, T const & ) + { + ar.finishNode(); + } + + // ###################################################################### + //! Prologue for arithmetic types for JSON archives + template inline + typename std::enable_if::value, void>::type + prologue( JSONOutputArchive & ar, T const & ) + { + ar.writeName(); + } + + //! Prologue for arithmetic types for JSON archives + template inline + typename std::enable_if::value, void>::type + prologue( JSONInputArchive &, T const & ) + { } + + // ###################################################################### + //! Epilogue for arithmetic types for JSON archives + template inline + typename std::enable_if::value, void>::type + epilogue( JSONOutputArchive &, T const & ) + { } + + //! Epilogue for arithmetic types for JSON archives + template inline + typename std::enable_if::value, void>::type + epilogue( JSONInputArchive &, T const & ) + { } + + // ###################################################################### + //! Prologue for strings for JSON archives + template inline + void prologue(JSONOutputArchive & ar, std::basic_string const &) + { + ar.writeName(); + } + + //! Prologue for strings for JSON archives + template inline + void prologue(JSONInputArchive &, std::basic_string const &) + { } + + // ###################################################################### + //! Epilogue for strings for JSON archives + template inline + void epilogue(JSONOutputArchive &, std::basic_string const &) + { } + + //! Epilogue for strings for JSON archives + template inline + void epilogue(JSONInputArchive &, std::basic_string const &) + { } + + // ###################################################################### + //! Prologue for std::optional<> types for JSON archives + template inline + void prologue(JSONOutputArchive & ar, std::optional const &) + { + ar.writeName(); + } + + //! Prologue for std::optional<> types for JSON archives + template inline + void prologue(JSONInputArchive &, std::optional const &) + { } + + // ###################################################################### + //! Epilogue for std::optional<> types for JSON archives + template inline + void epilogue(JSONOutputArchive &, std::optional const &) + { } + + //! Epilogue for std::optional<> types for JSON archives + template inline + void epilogue(JSONInputArchive &, std::optional const &) + { } + + // ###################################################################### + // Common JSONArchive serialization functions + // ###################################################################### + + //! Serializing NVP types to JSON + template inline + void save( JSONOutputArchive & ar, NameValuePair const & t ) + { + ar.setNextName( t.name ); + ar( t.value ); + } + + template inline + void load( JSONInputArchive & ar, NameValuePair & t ) + { + ar.setNextName( t.name ); + ar( t.value ); + } + + //! Saving for arithmetic to JSON + template inline + typename std::enable_if::value, void>::type + save(JSONOutputArchive & ar, T const & t) + { + ar.saveValue( t ); + } + + //! Loading arithmetic from JSON + template inline + typename std::enable_if::value, void>::type + load(JSONInputArchive & ar, T & t) + { + ar.loadValue( t ); + } + + //! saving string to JSON + template inline + void save(JSONOutputArchive & ar, std::basic_string const & str) + { + ar.saveValue( str ); + } + + //! loading string from JSON + template inline + void load(JSONInputArchive & ar, std::basic_string & str) + { + ar.loadValue( str ); + } + + // ###################################################################### + //! Saving SizeTags to JSON + template inline + void save( JSONOutputArchive &, SizeTag const & ) + { + // nothing to do here, we don't explicitly save the size + } + + //! Loading SizeTags from JSON + template inline + void load( JSONInputArchive & ar, SizeTag & st ) + { + ar.loadSize( st.size ); + } +} // namespace cereal + +// register archives for polymorphic support +CEREAL_REGISTER_ARCHIVE(cereal::JSONInputArchive) +CEREAL_REGISTER_ARCHIVE(cereal::JSONOutputArchive) + +#endif // CEREAL_ARCHIVES_JSON_HPP_ diff --git a/src/lib/helper/cereal/cereal_type_jsonvalue.h b/src/lib/helper/cereal/cereal_type_jsonvalue.h new file mode 100644 index 000000000..694bba3b0 --- /dev/null +++ b/src/lib/helper/cereal/cereal_type_jsonvalue.h @@ -0,0 +1,42 @@ +#ifndef CEREAL_TYPES_JSONVALUE_H_ +#define CEREAL_TYPES_JSONVALUE_H_ + +#include + + +static const int JSON_VALUE_TYPE_ARRAY = 4; + +namespace cereal +{ + //! Loading Json::Value + template inline + void load( Archive & ar, Json::Value & val ) + { + + int valueType = ar.valueType(); + + if (valueType == JSON_VALUE_TYPE_ARRAY) + { + Json::Reader reader; + size_type size; + std::vector values; + std::string str_val; + + ar( make_size_tag( size ) ); + + values.resize( static_cast( size ) ); + for( auto it = values.begin(), end = values.end(); it != end; ++it ) + { + str_val = ar.parseObjectToString(); + reader.parse(str_val, *it); + } + + for (auto jason_value : values) + val.append(jason_value); + } else { + std::cerr << "Error: Unable to serialize given type into Json::Value." << std::endl; + } + } +} // namespace cereal + +#endif // CEREAL_TYPES_JSONVALUE_H_ diff --git a/src/lib/helper/cereal/cereal_type_optional.h b/src/lib/helper/cereal/cereal_type_optional.h new file mode 100644 index 000000000..7a1b51dd5 --- /dev/null +++ b/src/lib/helper/cereal/cereal_type_optional.h @@ -0,0 +1,21 @@ +#ifndef CEREAL_TYPES_OPTIONAL_H_ +#define CEREAL_TYPES_OPTIONAL_H_ + +#include + +namespace cereal +{ + //! Loading std::optional + template inline + void load( Archive & ar, std::optional & opt ) + { + ar.setOptional(true); + T optValue; + ar( optValue ); + if (ar.foundOptionalValue()) opt = std::optional(optValue); + ar.setOptional(false); + } +} // namespace cereal + +#endif // CEREAL_TYPES_OPTIONAL_H_ + diff --git a/src/lib/helper/cereal/serialize_macro.h b/src/lib/helper/cereal/serialize_macro.h new file mode 100644 index 000000000..7853ffc86 --- /dev/null +++ b/src/lib/helper/cereal/serialize_macro.h @@ -0,0 +1,64 @@ +#ifndef SERIALIZE_MACRO_H_ +#define SERIALIZE_MACRO_H_ + + +#define CEREAL(...) template\ + void serialize( Archive & archive )\ + {\ + archive(\ + __VA_ARGS__\ + );\ + } + + +#define SERIALIZE_0() +#define SERIALIZE_1(A) CEREAL(cereal::make_nvp(#A, A)) +#define SERIALIZE_2(A,B) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B)) +#define SERIALIZE_3(A,B,C) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C)) +#define SERIALIZE_4(A,B,C,D) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D)) +#define SERIALIZE_5(A,B,C,D,E) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E)) +#define SERIALIZE_6(A,B,C,D,E,F) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F)) +#define SERIALIZE_7(A,B,C,D,E,F,G) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G)) +#define SERIALIZE_8(A,B,C,D,E,F,G,H) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H)) +#define SERIALIZE_9(A,B,C,D,E,F,G,H,I) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I)) +#define SERIALIZE_10(A,B,C,D,E,F,G,H,I,J) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J)) +#define SERIALIZE_11(A,B,C,D,E,F,G,H,I,J,K) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K)) +#define SERIALIZE_12(A,B,C,D,E,F,G,H,I,J,K,L) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L)) +#define SERIALIZE_13(A,B,C,D,E,F,G,H,I,J,K,L,M) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M)) +#define SERIALIZE_14(A,B,C,D,E,F,G,H,I,J,K,L,M,N) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N)) +#define SERIALIZE_15(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O)) +#define SERIALIZE_16(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P)) +#define SERIALIZE_17(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q)) +#define SERIALIZE_18(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q), cereal::make_nvp(#R, R)) +#define SERIALIZE_19(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q), cereal::make_nvp(#R, R), cereal::make_nvp(#S, S)) +#define SERIALIZE_20(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q), cereal::make_nvp(#R, R), cereal::make_nvp(#S, S), cereal::make_nvp(#T, T)) + +// The interim macro that simply strips the excess and ends up with the required macro +#define SERIALIZE_X(x,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,FUNC, ...) FUNC + +// The macro that the programmer uses +#define SERIALIZE(...) SERIALIZE_X(,##__VA_ARGS__,\ + SERIALIZE_20(__VA_ARGS__),\ + SERIALIZE_19(__VA_ARGS__),\ + SERIALIZE_18(__VA_ARGS__),\ + SERIALIZE_17(__VA_ARGS__),\ + SERIALIZE_16(__VA_ARGS__),\ + SERIALIZE_15(__VA_ARGS__),\ + SERIALIZE_14(__VA_ARGS__),\ + SERIALIZE_13(__VA_ARGS__),\ + SERIALIZE_12(__VA_ARGS__),\ + SERIALIZE_11(__VA_ARGS__),\ + SERIALIZE_10(__VA_ARGS__),\ + SERIALIZE_9(__VA_ARGS__),\ + SERIALIZE_8(__VA_ARGS__),\ + SERIALIZE_7(__VA_ARGS__),\ + SERIALIZE_6(__VA_ARGS__),\ + SERIALIZE_5(__VA_ARGS__),\ + SERIALIZE_4(__VA_ARGS__),\ + SERIALIZE_3(__VA_ARGS__),\ + SERIALIZE_2(__VA_ARGS__),\ + SERIALIZE_1(__VA_ARGS__),\ + SERIALIZE_0(__VA_ARGS__)\ + ) + +#endif // SERIALIZE_MACRO_H_ diff --git a/src/lib/helper/serialization.h b/src/lib/helper/serialization.h new file mode 100644 index 000000000..3f40befa0 --- /dev/null +++ b/src/lib/helper/serialization.h @@ -0,0 +1,11 @@ +// Copyright (c) 2012 Hasso-Plattner-Institut fuer Softwaresystemtechnik GmbH. All rights reserved. +#pragma once + + +#include "cereal/HyriseCerealJsonArchive.h" +#include "cereal/cereal_type_jsonvalue.h" +#include "cereal/cereal_type_optional.h" +#include "cereal/serialize_macro.h" + +#include +#include diff --git a/third_party/cereal b/third_party/cereal index d83b97706..30836ce2b 160000 --- a/third_party/cereal +++ b/third_party/cereal @@ -1 +1 @@ -Subproject commit d83b977062b06fca5b10ddb394f959d063e107cd +Subproject commit 30836ce2bf07cc93f68668a8c6d1ef6af29f0438 From 74149404fcfae5a6d12c2257813781b0d77acdd1 Mon Sep 17 00:00:00 2001 From: lanice Date: Thu, 3 Apr 2014 13:19:04 +0000 Subject: [PATCH 2/5] Add cereal_construct to Queryparser, add parameter struct and serialization to TableLoad and SimpleTableScan. --- src/bin/units_access/ops_select.cpp | 10 ++- src/lib/access/SimpleTableScan.cpp | 38 +++++------ src/lib/access/SimpleTableScan.h | 16 ++++- src/lib/access/storage/TableLoad.cpp | 67 +++++++++---------- src/lib/access/storage/TableLoad.h | 28 +++++--- src/lib/access/system/QueryParser.h | 25 +++++++ .../materialization_no_projection.json | 1 + test/autojson/merge.json | 4 +- 8 files changed, 117 insertions(+), 72 deletions(-) diff --git a/src/bin/units_access/ops_select.cpp b/src/bin/units_access/ops_select.cpp index 15eea50a2..5e8b47ba4 100644 --- a/src/bin/units_access/ops_select.cpp +++ b/src/bin/units_access/ops_select.cpp @@ -157,7 +157,15 @@ TEST_F(SelectTests, simple_expression) { TEST_F(SelectTests, should_throw_without_predicates) { Json::Value v(Json::objectValue); v["type"] = "SimpleTableScan"; - ASSERT_THROW(SimpleTableScan::parse(v), std::runtime_error); + + std::stringstream ss; + ss << v.toStyledString(); + + cereal::JSONInputArchive archive(ss); + + typename SimpleTableScan::Parameters params; + + ASSERT_THROW(params.serialize(archive), std::runtime_error); } diff --git a/src/lib/access/SimpleTableScan.cpp b/src/lib/access/SimpleTableScan.cpp index 39824f2d2..5652f09e8 100644 --- a/src/lib/access/SimpleTableScan.cpp +++ b/src/lib/access/SimpleTableScan.cpp @@ -8,14 +8,30 @@ #include "helper/checked_cast.h" +#include +#include +#include + namespace hyrise { namespace access { namespace { - auto _ = QueryParser::registerPlanOperation("SimpleTableScan"); + auto _ = QueryParser::registerSerializablePlanOperation("SimpleTableScan"); +} + +SimpleTableScan::SimpleTableScan(): + _comparator(nullptr), + _ofDelta(false) +{ } -SimpleTableScan::SimpleTableScan(): _comparator(nullptr) { +SimpleTableScan::SimpleTableScan(const Parameters & parameters): + _comparator(nullptr), + _ofDelta(false) +{ + setPredicate(buildExpression(parameters.predicates)); + if (parameters.materializing) setProducesPositions(!*parameters.materializing); + if (parameters.ofDelta) _ofDelta = *parameters.ofDelta; } SimpleTableScan::~SimpleTableScan() { @@ -71,24 +87,6 @@ void SimpleTableScan::executePlanOperation() { } } -std::shared_ptr SimpleTableScan::parse(const Json::Value &data) { - std::shared_ptr pop = std::make_shared(); - - if (data.isMember("materializing")) - pop->setProducesPositions(!data["materializing"].asBool()); - - if (!data.isMember("predicates")) { - throw std::runtime_error("There is no reason for a Selection without predicates"); - } - pop->setPredicate(buildExpression(data["predicates"])); - - if (data.isMember("ofDelta")) { - pop->_ofDelta = data["ofDelta"].asBool(); - } - - return pop; -} - const std::string SimpleTableScan::vname() { return "SimpleTableScan"; } diff --git a/src/lib/access/SimpleTableScan.h b/src/lib/access/SimpleTableScan.h index b933f0611..a610da66c 100644 --- a/src/lib/access/SimpleTableScan.h +++ b/src/lib/access/SimpleTableScan.h @@ -4,26 +4,38 @@ #include "access/system/ParallelizablePlanOperation.h" #include "access/expressions/pred_SimpleExpression.h" +#include "helper/serialization.h" namespace hyrise { namespace access { class SimpleTableScan : public ParallelizablePlanOperation { + +public: + struct Parameters + { + std::string type; + Json::Value predicates; + std::optional materializing, ofDelta; + + SERIALIZE(type, predicates, materializing, ofDelta) + }; + public: SimpleTableScan(); + SimpleTableScan(const Parameters & parameters); virtual ~SimpleTableScan(); void setupPlanOperation(); void executePlanOperation(); void executePositional(); void executeMaterialized(); - static std::shared_ptr parse(const Json::Value &data); const std::string vname(); void setPredicate(SimpleExpression *c); private: SimpleExpression *_comparator; - bool _ofDelta = false; + bool _ofDelta; }; } diff --git a/src/lib/access/storage/TableLoad.cpp b/src/lib/access/storage/TableLoad.cpp index 51e2a6a03..c167c30f9 100644 --- a/src/lib/access/storage/TableLoad.cpp +++ b/src/lib/access/storage/TableLoad.cpp @@ -13,13 +13,25 @@ namespace hyrise { namespace access { namespace { - auto _ = QueryParser::registerPlanOperation("TableLoad"); + auto _ = QueryParser::registerSerializablePlanOperation("TableLoad"); log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("access.plan.PlanOperation")); } -TableLoad::TableLoad(): _hasDelimiter(false), - _binary(false), - _unsafe(false), + +TableLoad::TableLoad(const Parameters & parameters): + _table_name(parameters.table), + _file_name(parameters.filename), + _header_file_name(parameters.header), + _header_string(parameters.header_string), + _delimiter(parameters.delimiter), + _unsafe(parameters.unsafe), + _raw(parameters.raw) + { + if (parameters.unsafe) _unsafe = *parameters.unsafe; else _unsafe = false; + if (parameters.raw) _raw = *parameters.raw; else _raw = false; + } + +TableLoad::TableLoad(): _unsafe(false), _raw(false) { } @@ -31,29 +43,29 @@ void TableLoad::executePlanOperation() { if (!sm->exists(_table_name)) { // Load Raw Table - if (_raw) { + if (*_raw) { io::Loader::params p; p.setHeader(io::CSVHeader(_file_name)); p.setInput(io::RawTableLoader(_file_name)); sm->loadTable(_table_name, p); - } else if (!_header_string.empty()) { + } else if (_header_string) { // Load based on header string - auto p = io::Loader::shortcuts::loadWithStringHeaderParams(_file_name, _header_string); + auto p = io::Loader::shortcuts::loadWithStringHeaderParams(_file_name, *_header_string); sm->loadTable(_table_name, p); - } else if (_header_file_name.empty()) { + } else if (!_header_file_name) { // Load only with single file sm->loadTableFile(_table_name, _file_name); - } else if ((!_table_name.empty()) && (!_file_name.empty()) && (!_header_file_name.empty())) { + } else if ((!_table_name.empty()) && (!_file_name.empty()) && (_header_file_name)) { // Load with dedicated header file io::Loader::params p; p.setCompressed(false); - p.setHeader(io::CSVHeader(_header_file_name)); - auto params = io::CSVInput::params().setUnsafe(_unsafe); - if (_hasDelimiter) - params.setCSVParams(io::csv::params().setDelimiter(_delimiter.at(0))); + p.setHeader(io::CSVHeader(*_header_file_name)); + auto params = io::CSVInput::params().setUnsafe(*_unsafe); + if (_delimiter) + params.setCSVParams(io::csv::params().setDelimiter((*_delimiter).at(0))); p.setInput(io::CSVInput(_file_name, params)); sm->loadTable(_table_name, p); } @@ -68,20 +80,6 @@ void TableLoad::executePlanOperation() { addResult(_table); } -std::shared_ptr TableLoad::parse(const Json::Value &data) { - std::shared_ptr s = std::make_shared(); - s->setTableName(data["table"].asString()); - s->setFileName(data["filename"].asString()); - s->setHeaderFileName(data["header"].asString()); - s->setHeaderString(data["header_string"].asString()); - s->setUnsafe(data["unsafe"].asBool()); - s->setRaw(data["raw"].asBool()); - if (data.isMember("delimiter")) { - s->setDelimiter(data["delimiter"].asString()); - } - return s; -} - const std::string TableLoad::vname() { return "TableLoad"; } @@ -95,28 +93,23 @@ void TableLoad::setFileName(const std::string &filename) { } void TableLoad::setHeaderFileName(const std::string &filename) { - _header_file_name = filename; + _header_file_name = std::optional(filename); } void TableLoad::setHeaderString(const std::string &header) { - _header_string = header; -} - -void TableLoad::setBinary(const bool binary) { - _binary = binary; + _header_string = std::optional(header); } void TableLoad::setUnsafe(const bool unsafe) { - _unsafe = unsafe; + _unsafe = std::optional(unsafe); } void TableLoad::setRaw(const bool raw) { - _raw = raw; + _raw = std::optional(raw); } void TableLoad::setDelimiter(const std::string &d) { - _delimiter = d; - _hasDelimiter = true; + _delimiter = std::optional(d); } } diff --git a/src/lib/access/storage/TableLoad.h b/src/lib/access/storage/TableLoad.h index 6ba076b59..9443f2535 100644 --- a/src/lib/access/storage/TableLoad.h +++ b/src/lib/access/storage/TableLoad.h @@ -3,6 +3,7 @@ #define SRC_LIB_ACCESS_TABLELOAD_H_ #include "access/system/PlanOperation.h" +#include "helper/serialization.h" namespace hyrise { namespace access { @@ -13,33 +14,40 @@ class LoadTests_simple_unloadall_op_Test; class TableLoad : public PlanOperation { friend class LoadTests_simple_load_op_Test; friend class LoadTests_simple_unloadall_op_Test; - + +public: + struct Parameters + { + std::string type, table, filename; + std::optional header, header_string, delimiter; + std::optional unsafe, raw; + + SERIALIZE(type, table, filename, header, header_string, delimiter, unsafe, raw) + }; + public: TableLoad(); + TableLoad(const Parameters & parameters); virtual ~TableLoad(); void executePlanOperation(); - static std::shared_ptr parse(const Json::Value &data); const std::string vname(); void setTableName(const std::string &tablename); void setFileName(const std::string &filename); void setHeaderFileName(const std::string &filename); void setHeaderString(const std::string &header); - void setBinary(const bool binary); void setUnsafe(const bool unsafe); void setRaw(const bool raw); void setDelimiter(const std::string &d); private: std::string _table_name; - std::string _header_file_name; std::string _file_name; - std::string _header_string; - std::string _delimiter; - bool _hasDelimiter; - bool _binary; - bool _unsafe; - bool _raw; + std::optional _header_file_name; + std::optional _header_string; + std::optional _delimiter; + std::optional _unsafe; + std::optional _raw; }; } diff --git a/src/lib/access/system/QueryParser.h b/src/lib/access/system/QueryParser.h index 13f6d0e60..542e0f8a6 100644 --- a/src/lib/access/system/QueryParser.h +++ b/src/lib/access/system/QueryParser.h @@ -8,9 +8,11 @@ #include #include #include +#include #include +#include "helper/cereal/HyriseCerealJsonArchive.h" #include "access/system/BasicParser.h" const std::string autojsonReferenceTableId = "-1"; @@ -39,6 +41,7 @@ struct AbstractQueryParserFactory { struct parse_construct {}; struct default_construct {}; +struct cereal_construct {}; template struct QueryParserFactory; @@ -59,6 +62,22 @@ struct QueryParserFactory : public AbstractQueryParserFact } }; +template +struct QueryParserFactory : public AbstractQueryParserFactory { + + virtual std::shared_ptr parse(const Json::Value& data) { + std::stringstream ss; + ss << data.toStyledString(); + + cereal::JSONInputArchive archive(ss); + + typename T::Parameters params; + params.serialize(archive); + + return std::make_shared(params); + } +}; + /* * The Query Parser parses a given Json Value to create a plan operation * @@ -116,6 +135,12 @@ class QueryParser { QueryParser::instance()._factory[name] = new QueryParserFactory(); return true; } + + template + static bool registerSerializablePlanOperation(const std::string& name) { + QueryParser::instance()._factory[name] = new QueryParserFactory(); + return true; + } std::shared_ptr parse(std::string name, const Json::Value& d); diff --git a/test/autojson/materialization_no_projection.json b/test/autojson/materialization_no_projection.json index feded36a3..269ab5ceb 100644 --- a/test/autojson/materialization_no_projection.json +++ b/test/autojson/materialization_no_projection.json @@ -2,6 +2,7 @@ "operators" : { "2": { "type": "TableLoad", + "table": "main", "filename" : "tables/companies.tbl", "name" : "companies" }, diff --git a/test/autojson/merge.json b/test/autojson/merge.json index 5e9370ff7..a0dafccef 100644 --- a/test/autojson/merge.json +++ b/test/autojson/merge.json @@ -11,12 +11,12 @@ }, "load_main" : { "type" : "TableLoad", - "table": "main", //TODO: Remove + "table": "main", "filename": "tables/employees.tbl" }, "load_delta" : { "type": "TableLoad", - "table": "delta", //TODO: Remove + "table": "delta", "filename": "tables/employees_new_row.tbl" }, "merge": { From b612f65dc09e36da596e9f2aaf1ff1387e1ab593 Mon Sep 17 00:00:00 2001 From: lanice Date: Thu, 3 Apr 2014 14:32:12 +0000 Subject: [PATCH 3/5] autoformat code --- src/bin/units_access/ops_select.cpp | 2 +- src/lib/access/SimpleTableScan.cpp | 19 +- src/lib/access/SimpleTableScan.h | 12 +- src/lib/access/storage/TableLoad.cpp | 62 +- src/lib/access/storage/TableLoad.h | 15 +- src/lib/access/system/QueryParser.h | 4 +- .../helper/cereal/HyriseCerealJsonArchive.h | 1562 ++++++++--------- src/lib/helper/cereal/cereal_type_jsonvalue.h | 58 +- src/lib/helper/cereal/cereal_type_optional.h | 28 +- src/lib/helper/cereal/serialize_macro.h | 308 +++- 10 files changed, 1075 insertions(+), 995 deletions(-) diff --git a/src/bin/units_access/ops_select.cpp b/src/bin/units_access/ops_select.cpp index 76cc6c634..8d946b91a 100644 --- a/src/bin/units_access/ops_select.cpp +++ b/src/bin/units_access/ops_select.cpp @@ -164,7 +164,7 @@ TEST_F(SelectTests, should_throw_without_predicates) { cereal::JSONInputArchive archive(ss); typename SimpleTableScan::Parameters params; - + ASSERT_THROW(params.serialize(archive), std::runtime_error); } diff --git a/src/lib/access/SimpleTableScan.cpp b/src/lib/access/SimpleTableScan.cpp index ad80f4556..cfadb4733 100644 --- a/src/lib/access/SimpleTableScan.cpp +++ b/src/lib/access/SimpleTableScan.cpp @@ -16,22 +16,17 @@ namespace hyrise { namespace access { namespace { - auto _ = QueryParser::registerSerializablePlanOperation("SimpleTableScan"); +auto _ = QueryParser::registerSerializablePlanOperation("SimpleTableScan"); } -SimpleTableScan::SimpleTableScan(): - _comparator(nullptr), - _ofDelta(false) -{ -} +SimpleTableScan::SimpleTableScan() : _comparator(nullptr), _ofDelta(false) {} -SimpleTableScan::SimpleTableScan(const Parameters & parameters): - _comparator(nullptr), - _ofDelta(false) -{ +SimpleTableScan::SimpleTableScan(const Parameters& parameters) : _comparator(nullptr), _ofDelta(false) { setPredicate(buildExpression(parameters.predicates)); - if (parameters.materializing) setProducesPositions(!*parameters.materializing); - if (parameters.ofDelta) _ofDelta = *parameters.ofDelta; + if (parameters.materializing) + setProducesPositions(!*parameters.materializing); + if (parameters.ofDelta) + _ofDelta = *parameters.ofDelta; } SimpleTableScan::~SimpleTableScan() { diff --git a/src/lib/access/SimpleTableScan.h b/src/lib/access/SimpleTableScan.h index 1bee22048..3f2518476 100644 --- a/src/lib/access/SimpleTableScan.h +++ b/src/lib/access/SimpleTableScan.h @@ -11,9 +11,8 @@ namespace access { class SimpleTableScan : public ParallelizablePlanOperation { -public: - struct Parameters - { + public: + struct Parameters { std::string type; Json::Value predicates; std::optional materializing, ofDelta; @@ -21,9 +20,9 @@ class SimpleTableScan : public ParallelizablePlanOperation { SERIALIZE(type, predicates, materializing, ofDelta) }; -public: + public: SimpleTableScan(); - SimpleTableScan(const Parameters & parameters); + SimpleTableScan(const Parameters& parameters); virtual ~SimpleTableScan(); void setupPlanOperation(); @@ -34,10 +33,9 @@ class SimpleTableScan : public ParallelizablePlanOperation { const std::string vname(); void setPredicate(SimpleExpression* c); -private: + private: SimpleExpression* _comparator; bool _ofDelta; - }; } } diff --git a/src/lib/access/storage/TableLoad.cpp b/src/lib/access/storage/TableLoad.cpp index 34814169f..32cc75f45 100644 --- a/src/lib/access/storage/TableLoad.cpp +++ b/src/lib/access/storage/TableLoad.cpp @@ -16,25 +16,33 @@ namespace hyrise { namespace access { namespace { - auto _ = QueryParser::registerSerializablePlanOperation("TableLoad"); - log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("access.plan.PlanOperation")); +auto _ = QueryParser::registerSerializablePlanOperation("TableLoad"); +log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("access.plan.PlanOperation")); } -TableLoad::TableLoad(const Parameters & parameters): - _table_name(parameters.table), - _file_name(parameters.filename), - _header_file_name(parameters.header), - _header_string(parameters.header_string), - _delimiter(parameters.delimiter), - _path(parameters.path), - _unsafe(parameters.unsafe), - _raw(parameters.raw) - { - if (parameters.path) _path = *parameters.path; else _path = std::string(""); - if (parameters.unsafe) _unsafe = *parameters.unsafe; else _unsafe = false; - if (parameters.raw) _raw = *parameters.raw; else _raw = false; - } +TableLoad::TableLoad(const Parameters& parameters) + : _table_name(parameters.table), + _file_name(parameters.filename), + _header_file_name(parameters.header), + _header_string(parameters.header_string), + _delimiter(parameters.delimiter), + _path(parameters.path), + _unsafe(parameters.unsafe), + _raw(parameters.raw) { + if (parameters.path) + _path = *parameters.path; + else + _path = std::string(""); + if (parameters.unsafe) + _unsafe = *parameters.unsafe; + else + _unsafe = false; + if (parameters.raw) + _raw = *parameters.raw; + else + _raw = false; +} TableLoad::TableLoad() : _path(std::string("")), _unsafe(false), _raw(false), _nonvolatile(false), _binary(false) {} @@ -93,28 +101,18 @@ void TableLoad::setTableName(const std::string& tablename) { _table_name = table void TableLoad::setFileName(const std::string& filename) { _file_name = filename; } -void TableLoad::setHeaderFileName(const std::string &filename) { +void TableLoad::setHeaderFileName(const std::string& filename) { _header_file_name = std::optional(filename); } -void TableLoad::setHeaderString(const std::string &header) { - _header_string = std::optional(header); -} +void TableLoad::setHeaderString(const std::string& header) { _header_string = std::optional(header); } -void TableLoad::setUnsafe(const bool unsafe) { - _unsafe = std::optional(unsafe); -} +void TableLoad::setUnsafe(const bool unsafe) { _unsafe = std::optional(unsafe); } -void TableLoad::setRaw(const bool raw) { - _raw = std::optional(raw); -} +void TableLoad::setRaw(const bool raw) { _raw = std::optional(raw); } -void TableLoad::setDelimiter(const std::string &d) { - _delimiter = std::optional(d); -} -void TableLoad::setPath(const std::string& path) { - _path = std::optional(path); -} +void TableLoad::setDelimiter(const std::string& d) { _delimiter = std::optional(d); } +void TableLoad::setPath(const std::string& path) { _path = std::optional(path); } void TableLoad::setBinary(const bool binary) { _binary = binary; } diff --git a/src/lib/access/storage/TableLoad.h b/src/lib/access/storage/TableLoad.h index cc37ef9f6..9f6271195 100644 --- a/src/lib/access/storage/TableLoad.h +++ b/src/lib/access/storage/TableLoad.h @@ -14,24 +14,23 @@ class LoadTests_simple_unloadall_op_Test; class TableLoad : public PlanOperation { friend class LoadTests_simple_load_op_Test; friend class LoadTests_simple_unloadall_op_Test; - -public: - struct Parameters - { + + public: + struct Parameters { std::string type, table, filename; std::optional header, header_string, delimiter, path; std::optional unsafe, raw; SERIALIZE(type, table, filename, header, header_string, delimiter, path, unsafe, raw) }; - -public: + + public: TableLoad(); - TableLoad(const Parameters & parameters); + TableLoad(const Parameters& parameters); virtual ~TableLoad(); void executePlanOperation(); - + const std::string vname(); void setTableName(const std::string& tablename); void setFileName(const std::string& filename); diff --git a/src/lib/access/system/QueryParser.h b/src/lib/access/system/QueryParser.h index f03aa6c6e..073b0e862 100644 --- a/src/lib/access/system/QueryParser.h +++ b/src/lib/access/system/QueryParser.h @@ -59,7 +59,7 @@ struct QueryParserFactory : public AbstractQueryParserFact } }; -template +template struct QueryParserFactory : public AbstractQueryParserFactory { virtual std::shared_ptr parse(const Json::Value& data) { @@ -127,7 +127,7 @@ class QueryParser { return true; } - template + template static bool registerSerializablePlanOperation(const std::string& name) { QueryParser::instance()._factory[name] = new QueryParserFactory(); return true; diff --git a/src/lib/helper/cereal/HyriseCerealJsonArchive.h b/src/lib/helper/cereal/HyriseCerealJsonArchive.h index 51f617543..5d0f15122 100644 --- a/src/lib/helper/cereal/HyriseCerealJsonArchive.h +++ b/src/lib/helper/cereal/HyriseCerealJsonArchive.h @@ -48,903 +48,799 @@ #include #include -namespace cereal -{ - // ###################################################################### - //! An output archive designed to save data to JSON - /*! This archive uses RapidJSON to build serialie data to JSON. - - JSON archives provides a human readable output but at decreased - performance (both in time and space) compared to binary archives. - - JSON benefits greatly from name-value pairs, which if present, will - name the nodes in the output. If these are not present, each level - of the output will be given an automatically generated delimited name. - - The precision of the output archive controls the number of decimals output - for floating point numbers and should be sufficiently large (i.e. at least 20) - if there is a desire to have binary equality between the numbers output and - those read in. In general you should expect a loss of precision when going - from floating point to text and back. - - JSON archives do not output the size information for any dynamically sized structure - and instead infer it from the number of children for a node. This means that data - can be hand edited for dynamic sized structures and will still be readable. This - is accomplished through the cereal::SizeTag object, which will cause the archive - to output the data as a JSON array (e.g. marked by [] instead of {}), which indicates - that the container is variable sized and may be edited. - - \ingroup Archives */ - class JSONOutputArchive : public OutputArchive - { - enum class NodeType { StartObject, InObject, StartArray, InArray }; - - typedef rapidjson::GenericWriteStream WriteStream; - typedef rapidjson::PrettyWriter JSONWriter; - - public: - /*! @name Common Functionality - Common use cases for directly interacting with an JSONOutputArchive */ - //! @{ - - //! Construct, outputting to the provided stream - /*! @param stream The stream to output to. Can be a stringstream, a file stream, or - even cout! - @param precision The precision for floating point output */ - JSONOutputArchive(std::ostream & stream, int precision = std::numeric_limits::max_digits10) : - OutputArchive(this), +namespace cereal { +// ###################################################################### +//! An output archive designed to save data to JSON +/*! This archive uses RapidJSON to build serialie data to JSON. + + JSON archives provides a human readable output but at decreased + performance (both in time and space) compared to binary archives. + + JSON benefits greatly from name-value pairs, which if present, will + name the nodes in the output. If these are not present, each level + of the output will be given an automatically generated delimited name. + + The precision of the output archive controls the number of decimals output + for floating point numbers and should be sufficiently large (i.e. at least 20) + if there is a desire to have binary equality between the numbers output and + those read in. In general you should expect a loss of precision when going + from floating point to text and back. + + JSON archives do not output the size information for any dynamically sized structure + and instead infer it from the number of children for a node. This means that data + can be hand edited for dynamic sized structures and will still be readable. This + is accomplished through the cereal::SizeTag object, which will cause the archive + to output the data as a JSON array (e.g. marked by [] instead of {}), which indicates + that the container is variable sized and may be edited. + + \ingroup Archives */ +class JSONOutputArchive : public OutputArchive { + enum class NodeType { + StartObject, + InObject, + StartArray, + InArray + }; + + typedef rapidjson::GenericWriteStream WriteStream; + typedef rapidjson::PrettyWriter JSONWriter; + + public: + /*! @name Common Functionality + Common use cases for directly interacting with an JSONOutputArchive */ + //! @{ + + //! Construct, outputting to the provided stream + /*! @param stream The stream to output to. Can be a stringstream, a file stream, or + even cout! + @param precision The precision for floating point output */ + JSONOutputArchive(std::ostream& stream, int precision = std::numeric_limits::max_digits10) + : OutputArchive(this), itsWriteStream(stream), itsWriter(itsWriteStream, precision), - itsNextName(nullptr) - { - itsNameCounter.push(0); - itsNodeStack.push(NodeType::StartObject); - } + itsNextName(nullptr) { + itsNameCounter.push(0); + itsNodeStack.push(NodeType::StartObject); + } - //! Destructor, flushes the JSON - ~JSONOutputArchive() - { - itsWriter.EndObject(); - } + //! Destructor, flushes the JSON + ~JSONOutputArchive() { itsWriter.EndObject(); } - //! Saves some binary data, encoded as a base64 string, with an optional name - /*! This will create a new node, optionally named, and insert a value that consists of - the data encoded as a base64 string */ - void saveBinaryValue( const void * data, size_t size, const char * name = nullptr ) - { - setNextName( name ); - writeName(); - - auto base64string = base64::encode( reinterpret_cast( data ), size ); - saveValue( base64string ); - }; - - //! @} - /*! @name Internal Functionality - Functionality designed for use by those requiring control over the inner mechanisms of - the JSONOutputArchive */ - //! @{ - - //! Starts a new node in the JSON output - /*! The node can optionally be given a name by calling setNextName prior - to creating the node - - Nodes only need to be started for types that are themselves objects or arrays */ - void startNode() - { - writeName(); - itsNodeStack.push(NodeType::StartObject); - itsNameCounter.push(0); - } + //! Saves some binary data, encoded as a base64 string, with an optional name + /*! This will create a new node, optionally named, and insert a value that consists of + the data encoded as a base64 string */ + void saveBinaryValue(const void* data, size_t size, const char* name = nullptr) { + setNextName(name); + writeName(); - //! Designates the most recently added node as finished - void finishNode() - { - // if we ended up serializing an empty object or array, writeName - // will never have been called - so start and then immediately end - // the object/array. - // - // We'll also end any object/arrays we happen to be in - switch(itsNodeStack.top()) - { - case NodeType::StartArray: - itsWriter.StartArray(); - case NodeType::InArray: - itsWriter.EndArray(); - break; - case NodeType::StartObject: - itsWriter.StartObject(); - case NodeType::InObject: - itsWriter.EndObject(); - break; - } + auto base64string = base64::encode(reinterpret_cast(data), size); + saveValue(base64string); + }; - itsNodeStack.pop(); - itsNameCounter.pop(); - } + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the JSONOutputArchive */ + //! @{ + + //! Starts a new node in the JSON output + /*! The node can optionally be given a name by calling setNextName prior + to creating the node + + Nodes only need to be started for types that are themselves objects or arrays */ + void startNode() { + writeName(); + itsNodeStack.push(NodeType::StartObject); + itsNameCounter.push(0); + } - //! Sets the name for the next node created with startNode - void setNextName( const char * name ) - { - itsNextName = name; - } + //! Designates the most recently added node as finished + void finishNode() { + // if we ended up serializing an empty object or array, writeName + // will never have been called - so start and then immediately end + // the object/array. + // + // We'll also end any object/arrays we happen to be in + switch (itsNodeStack.top()) { + case NodeType::StartArray: + itsWriter.StartArray(); + case NodeType::InArray: + itsWriter.EndArray(); + break; + case NodeType::StartObject: + itsWriter.StartObject(); + case NodeType::InObject: + itsWriter.EndObject(); + break; + } - //! Saves a bool to the current node - void saveValue(bool b) { itsWriter.Bool(b); } - //! Saves an int to the current node - void saveValue(int i) { itsWriter.Int(i); } - //! Saves a uint to the current node - void saveValue(unsigned u) { itsWriter.Uint(u); } - //! Saves an int64 to the current node - void saveValue(int64_t i64) { itsWriter.Int64(i64); } - //! Saves a uint64 to the current node - void saveValue(uint64_t u64) { itsWriter.Uint64(u64); } - //! Saves a double to the current node - void saveValue(double d) { itsWriter.Double(d); } - //! Saves a string to the current node - void saveValue(std::string const & s) { itsWriter.String(s.c_str(), static_cast( s.size() )); } - //! Saves a const char * to the current node - void saveValue(char const * s) { itsWriter.String(s); } + itsNodeStack.pop(); + itsNameCounter.pop(); + } -#ifdef _MSC_VER - // Visual Studio has problems disambiguating the above for unsigned long, so we provide an explicit - // overload for long and serialize it as its size necessitates - // - // When loading we don't need to do this specialization since we catch the types with - // templates according to their size - - //! 32 bit long saving to current node - template inline - typename std::enable_if::type - saveLong(T lu){ saveValue( static_cast( lu ) ); } - - //! non 32 bit long saving to current node - template inline - typename std::enable_if::type - saveLong(T lu){ saveValue( static_cast( lu ) ); } - - //! MSVC only long overload to current node - void saveValue( unsigned long lu ){ saveLong( lu ); }; -#endif + //! Sets the name for the next node created with startNode + void setNextName(const char* name) { itsNextName = name; } + + //! Saves a bool to the current node + void saveValue(bool b) { itsWriter.Bool(b); } + //! Saves an int to the current node + void saveValue(int i) { itsWriter.Int(i); } + //! Saves a uint to the current node + void saveValue(unsigned u) { itsWriter.Uint(u); } + //! Saves an int64 to the current node + void saveValue(int64_t i64) { itsWriter.Int64(i64); } + //! Saves a uint64 to the current node + void saveValue(uint64_t u64) { itsWriter.Uint64(u64); } + //! Saves a double to the current node + void saveValue(double d) { itsWriter.Double(d); } + //! Saves a string to the current node + void saveValue(std::string const& s) { itsWriter.String(s.c_str(), static_cast(s.size())); } + //! Saves a const char * to the current node + void saveValue(char const* s) { itsWriter.String(s); } - //! Save exotic arithmetic types as binary to current node - template inline - typename std::enable_if::value && - (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), void>::type - saveValue(T const & t) - { - auto base64string = base64::encode( reinterpret_cast( &t ), sizeof(T) ); - saveValue( base64string ); - } +#ifdef _MSC_VER + // Visual Studio has problems disambiguating the above for unsigned long, so we provide an explicit + // overload for long and serialize it as its size necessitates + // + // When loading we don't need to do this specialization since we catch the types with + // templates according to their size + + //! 32 bit long saving to current node + template + inline typename std::enable_if::type saveLong(T lu) { + saveValue(static_cast(lu)); + } - //! Write the name of the upcoming node and prepare object/array state - /*! Since writeName is called for every value that is output, regardless of - whether it has a name or not, it is the place where we will do a deferred - check of our node state and decide whether we are in an array or an object. - - The general workflow of saving to the JSON archive is: - - 1. (optional) Set the name for the next node to be created, usually done by an NVP - 2. Start the node - 3. (if there is data to save) Write the name of the node (this function) - 4. (if there is data to save) Save the data (with saveValue) - 5. Finish the node - */ - void writeName() - { - NodeType const & nodeType = itsNodeStack.top(); - - // Start up either an object or an array, depending on state - if(nodeType == NodeType::StartArray) - { - itsWriter.StartArray(); - itsNodeStack.top() = NodeType::InArray; - } - else if(nodeType == NodeType::StartObject) - { - itsNodeStack.top() = NodeType::InObject; - itsWriter.StartObject(); - } + //! non 32 bit long saving to current node + template + inline typename std::enable_if::type saveLong(T lu) { + saveValue(static_cast(lu)); + } - // Array types do not output names - if(nodeType == NodeType::InArray) return; + //! MSVC only long overload to current node + void saveValue(unsigned long lu) { + saveLong(lu); + }; +#endif - if(itsNextName == nullptr) - { - std::string name = "value" + std::to_string( itsNameCounter.top()++ ) + "\0"; - saveValue(name); - } - else - { - saveValue(itsNextName); - itsNextName = nullptr; - } - } + //! Save exotic arithmetic types as binary to current node + template + inline typename std::enable_if< + std::is_arithmetic::value&&(sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), + void>::type + saveValue(T const& t) { + auto base64string = base64::encode(reinterpret_cast(&t), sizeof(T)); + saveValue(base64string); + } - //! Designates that the current node should be output as an array, not an object - void makeArray() - { - itsNodeStack.top() = NodeType::StartArray; - } + //! Write the name of the upcoming node and prepare object/array state + /*! Since writeName is called for every value that is output, regardless of + whether it has a name or not, it is the place where we will do a deferred + check of our node state and decide whether we are in an array or an object. + + The general workflow of saving to the JSON archive is: + + 1. (optional) Set the name for the next node to be created, usually done by an NVP + 2. Start the node + 3. (if there is data to save) Write the name of the node (this function) + 4. (if there is data to save) Save the data (with saveValue) + 5. Finish the node + */ + void writeName() { + NodeType const& nodeType = itsNodeStack.top(); + + // Start up either an object or an array, depending on state + if (nodeType == NodeType::StartArray) { + itsWriter.StartArray(); + itsNodeStack.top() = NodeType::InArray; + } else if (nodeType == NodeType::StartObject) { + itsNodeStack.top() = NodeType::InObject; + itsWriter.StartObject(); + } - //! @} - private: - WriteStream itsWriteStream; //!< Rapidjson write stream - JSONWriter itsWriter; //!< Rapidjson writer - char const * itsNextName; //!< The next name - std::stack itsNameCounter; //!< Counter for creating unique names for unnamed nodes - std::stack itsNodeStack; - }; // JSONOutputArchive - - // ###################################################################### - //! An input archive designed to load data from JSON - /*! This archive uses RapidJSON to read in a JSON archive. - - Input JSON should have been produced by the JSONOutputArchive. Data can - only be added to dynamically sized containers (marked by JSON arrays) - - the input archive will determine their size by looking at the number of child nodes. - Only JSON originating from a JSONOutputArchive is officially supported, but data - from other sources may work if properly formatted. - - The JSONInputArchive does not require that nodes are loaded in the same - order they were saved by JSONOutputArchive. Using name value pairs (NVPs), - it is possible to load in an out of order fashion or otherwise skip/select - specific nodes to load. - - The default behavior of the input archive is to read sequentially starting - with the first node and exploring its children. When a given NVP does - not match the read in name for a node, the archive will search for that - node at the current level and load it if it exists. After loading an out of - order node, the archive will then proceed back to loading sequentially from - its new position. - - Consider this simple example where loading of some data is skipped: - - @code{cpp} - // imagine the input file has someData(1-9) saved in order at the top level node - ar( someData1, someData2, someData3 ); // XML loads in the order it sees in the file - ar( cereal::make_nvp( "hello", someData6 ) ); // NVP given does not - // match expected NVP name, so we search - // for the given NVP and load that value - ar( someData7, someData8, someData9 ); // with no NVP given, loading resumes at its - // current location, proceeding sequentially - @endcode - - \ingroup Archives */ - class JSONInputArchive : public InputArchive - { - private: - typedef rapidjson::GenericReadStream ReadStream; - typedef rapidjson::GenericValue> JSONValue; - typedef JSONValue::ConstMemberIterator MemberIterator; - typedef JSONValue::ConstValueIterator ValueIterator; - typedef rapidjson::Document::GenericValue GenericValue; - - public: - /*! @name Common Functionality - Common use cases for directly interacting with an JSONInputArchive */ - //! @{ - - //! Construct, reading from the provided stream - /*! @param stream The stream to read from */ - JSONInputArchive(std::istream & stream) : - InputArchive(this), - itsNextName( nullptr ), - itsReadStream(stream) - { - itsDocument.ParseStream<0>(itsReadStream); - itsIteratorStack.emplace_back(itsDocument.MemberBegin(), itsDocument.MemberEnd()); - } + // Array types do not output names + if (nodeType == NodeType::InArray) + return; - //! Loads some binary data, encoded as a base64 string - /*! This will automatically start and finish a node to load the data, and can be called directly by - users. - - Note that this follows the same ordering rules specified in the class description in regards - to loading in/out of order */ - - void loadBinaryValue( void * data, size_t size, const char * name = nullptr ) - { - itsNextName = name; - - std::string encoded; - loadValue( encoded ); - auto decoded = base64::decode( encoded ); - - if( size != decoded.size() ) - throw Exception("Decoded binary data size does not match specified size"); - - std::memcpy( data, decoded.data(), decoded.size() ); - itsNextName = nullptr; - }; - - private: - //! @} - /*! @name Internal Functionality - Functionality designed for use by those requiring control over the inner mechanisms of - the JSONInputArchive */ - //! @{ - - //! An internal iterator that handles both array and object types - /*! This class is a variant and holds both types of iterators that - rapidJSON supports - one for arrays and one for objects. */ - class Iterator - { - public: - Iterator() : itsIndex( 0 ), itsType(Null) {} - - Iterator(MemberIterator begin, MemberIterator end) : - itsMemberItBegin(begin), itsMemberItEnd(end), itsIndex(0), itsType(Member) - { } - - Iterator(ValueIterator begin, ValueIterator end) : - itsValueItBegin(begin), itsValueItEnd(end), itsIndex(0), itsType(Value) - { } - - //! Advance to the next node - Iterator & operator++() - { - ++itsIndex; - return *this; - } - - //! Get the value of the current node - GenericValue const & value() - { - switch(itsType) - { - case Value : return itsValueItBegin[itsIndex]; - case Member: return itsMemberItBegin[itsIndex].value; - default: throw cereal::Exception("Invalid Iterator Type!"); - } - } - - //! Get the name of the current node, or nullptr if it has no name - const char * name() const - { - if( itsType == Member && (itsMemberItBegin + itsIndex) != itsMemberItEnd ) - return itsMemberItBegin[itsIndex].name.GetString(); - else - return nullptr; - } - - //! Adjust our position such that we are at the node with the given name - /*! @throws Exception if no such named node exists */ - inline void search( const char * searchName )//, GenericValue const & parent ) - { - size_t index = 0; - for( auto it = itsMemberItBegin; it != itsMemberItEnd; ++it, ++index ) - { - if( std::strcmp( searchName, it->name.GetString() ) == 0 ) - { - itsIndex = index; - return; - } - } - - if( _optional ) - { - itsIndex = -1; - return; - } - - throw std::runtime_error("JSON Parsing failed - provided NVP not found"); - } - - bool found() - { - return itsIndex != size_t(-1); - } - - void setOptional(bool opt) - { - _optional = opt; - } - - private: - MemberIterator itsMemberItBegin, itsMemberItEnd; //!< The member iterator (object) - ValueIterator itsValueItBegin, itsValueItEnd; //!< The value iterator (array) - public: - size_t itsIndex; //!< The current index of this iterator - private: - enum Type {Value, Member, Null} itsType; //!< Whether this holds values (array) or members (objects) or nothing - bool _optional = false; - }; - - //! Searches for the expectedName node if it doesn't match the actualName - /*! This needs to be called before every load or node start occurs. This function will - check to see if an NVP has been provided (with setNextName) and if so, see if that name matches the actual - next name given. If the names do not match, it will search in the current level of the JSON for that name. - If the name is not found, an exception will be thrown. - - Resets the NVP name after called. - - @throws Exception if an expectedName is given and not found */ - inline void search() - { - // The name an NVP provided with setNextName() - if( itsNextName ) - { - // The actual name of the current node - auto const actualName = itsIteratorStack.back().name(); - - // Do a search if we don't see a name coming up, or if the names don't match - if( !actualName || std::strcmp( itsNextName, actualName ) != 0 ) - itsIteratorStack.back().search( itsNextName ); - } + if (itsNextName == nullptr) { + std::string name = "value" + std::to_string(itsNameCounter.top()++) + "\0"; + saveValue(name); + } else { + saveValue(itsNextName); + itsNextName = nullptr; + } + } - itsNextName = nullptr; - } + //! Designates that the current node should be output as an array, not an object + void makeArray() { itsNodeStack.top() = NodeType::StartArray; } + + //! @} + private: + WriteStream itsWriteStream; //!< Rapidjson write stream + JSONWriter itsWriter; //!< Rapidjson writer + char const* itsNextName; //!< The next name + std::stack itsNameCounter; //!< Counter for creating unique names for unnamed nodes + std::stack itsNodeStack; +}; // JSONOutputArchive + +// ###################################################################### +//! An input archive designed to load data from JSON +/*! This archive uses RapidJSON to read in a JSON archive. + + Input JSON should have been produced by the JSONOutputArchive. Data can + only be added to dynamically sized containers (marked by JSON arrays) - + the input archive will determine their size by looking at the number of child nodes. + Only JSON originating from a JSONOutputArchive is officially supported, but data + from other sources may work if properly formatted. + + The JSONInputArchive does not require that nodes are loaded in the same + order they were saved by JSONOutputArchive. Using name value pairs (NVPs), + it is possible to load in an out of order fashion or otherwise skip/select + specific nodes to load. + + The default behavior of the input archive is to read sequentially starting + with the first node and exploring its children. When a given NVP does + not match the read in name for a node, the archive will search for that + node at the current level and load it if it exists. After loading an out of + order node, the archive will then proceed back to loading sequentially from + its new position. + + Consider this simple example where loading of some data is skipped: + + @code{cpp} + // imagine the input file has someData(1-9) saved in order at the top level node + ar( someData1, someData2, someData3 ); // XML loads in the order it sees in the file + ar( cereal::make_nvp( "hello", someData6 ) ); // NVP given does not + // match expected NVP name, so we search + // for the given NVP and load that value + ar( someData7, someData8, someData9 ); // with no NVP given, loading resumes at its + // current location, proceeding sequentially + @endcode + + \ingroup Archives */ +class JSONInputArchive : public InputArchive { + private: + typedef rapidjson::GenericReadStream ReadStream; + typedef rapidjson::GenericValue> JSONValue; + typedef JSONValue::ConstMemberIterator MemberIterator; + typedef JSONValue::ConstValueIterator ValueIterator; + typedef rapidjson::Document::GenericValue GenericValue; + + public: + /*! @name Common Functionality + Common use cases for directly interacting with an JSONInputArchive */ + //! @{ + + //! Construct, reading from the provided stream + /*! @param stream The stream to read from */ + JSONInputArchive(std::istream& stream) + : InputArchive(this), itsNextName(nullptr), itsReadStream(stream) { + itsDocument.ParseStream<0>(itsReadStream); + itsIteratorStack.emplace_back(itsDocument.MemberBegin(), itsDocument.MemberEnd()); + } - public: - //! Starts a new node, going into its proper iterator - /*! This places an iterator for the next node to be parsed onto the iterator stack. If the next - node is an array, this will be a value iterator, otherwise it will be a member iterator. - - By default our strategy is to start with the document root node and then recursively iterate through - all children in the order they show up in the document. - We don't need to know NVPs to do this; we'll just blindly load in the order things appear in. - - If we were given an NVP, we will search for it if it does not match our the name of the next node - that would normally be loaded. This functionality is provided by search(). */ - void startNode() - { - search(); - - if(itsIteratorStack.back().value().IsArray()) - itsIteratorStack.emplace_back(itsIteratorStack.back().value().Begin(), itsIteratorStack.back().value().End()); - else - itsIteratorStack.emplace_back(itsIteratorStack.back().value().MemberBegin(), itsIteratorStack.back().value().MemberEnd()); - } + //! Loads some binary data, encoded as a base64 string + /*! This will automatically start and finish a node to load the data, and can be called directly by + users. - //! Finishes the most recently started node - void finishNode() - { - itsIteratorStack.pop_back(); - ++itsIteratorStack.back(); - } + Note that this follows the same ordering rules specified in the class description in regards + to loading in/out of order */ - //! Sets the name for the next node created with startNode - void setNextName( const char * name ) - { - itsNextName = name; - } + void loadBinaryValue(void* data, size_t size, const char* name = nullptr) { + itsNextName = name; - //! Loads a value from the current node - small signed overload - template inline - typename std::enable_if::value && sizeof(T) < sizeof(int64_t), void>::type - loadValue(T & val) - { - search(); - - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetInt(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } + std::string encoded; + loadValue(encoded); + auto decoded = base64::decode(encoded); - //! Loads a value from the current node - small unsigned overload - template inline - typename std::enable_if<(std::is_unsigned::value && sizeof(T) < sizeof(uint64_t)) && - !std::is_same::value, void>::type - loadValue(T & val) - { - search(); - - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetUint(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } + if (size != decoded.size()) + throw Exception("Decoded binary data size does not match specified size"); - //! Loads a value from the current node - bool overload - void loadValue(bool & val) - { - search(); - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetBool(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } - //! Loads a value from the current node - int64 overload - void loadValue(int64_t & val) - { - search(); - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetInt64(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } - //! Loads a value from the current node - uint64 overload - void loadValue(uint64_t & val) - { - search(); - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetUint64(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } - //! Loads a value from the current node - float overload - void loadValue(float & val) - { - search(); - if ( itsIteratorStack.back().found()) - { - val = static_cast(itsIteratorStack.back().value().GetDouble()); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } - //! Loads a value from the current node - double overload - void loadValue(double & val) - { - search(); - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetDouble(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } - //! Loads a value from the current node - string overload - void loadValue(std::string & val) - { - search(); - if ( itsIteratorStack.back().found()) - { - val = itsIteratorStack.back().value().GetString(); - _foundOptionalValue = true; - } else { - _foundOptionalValue = false; - } - ++itsIteratorStack.back(); - } - //! Loads a value from the current node - long double and long long overloads - /*! These data types will automatically be encoded as base64 strings */ - template inline - typename std::enable_if::value && - (sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), void>::type - loadValue(T & val) - { - std::string encoded; - loadValue( encoded ); - auto decoded = base64::decode( encoded ); - - if( sizeof(T) != decoded.size() ) - throw Exception("Decoded binary data size does not match specified size"); - - std::memcpy( &val, decoded.data(), decoded.size() ); - } + std::memcpy(data, decoded.data(), decoded.size()); + itsNextName = nullptr; + }; - //! Loads the size for a SizeTag - void loadSize(size_type & size) - { - size = (itsIteratorStack.rbegin() + 1)->value().Size(); - } + private: + //! @} + /*! @name Internal Functionality + Functionality designed for use by those requiring control over the inner mechanisms of + the JSONInputArchive */ + //! @{ + + //! An internal iterator that handles both array and object types + /*! This class is a variant and holds both types of iterators that + rapidJSON supports - one for arrays and one for objects. */ + class Iterator { + public: + Iterator() : itsIndex(0), itsType(Null) {} + + Iterator(MemberIterator begin, MemberIterator end) + : itsMemberItBegin(begin), itsMemberItEnd(end), itsIndex(0), itsType(Member) {} + + Iterator(ValueIterator begin, ValueIterator end) + : itsValueItBegin(begin), itsValueItEnd(end), itsIndex(0), itsType(Value) {} + + //! Advance to the next node + Iterator& operator++() { + ++itsIndex; + return *this; + } - int valueType() - { - return (itsIteratorStack.rbegin() + 1)->value().GetType(); + //! Get the value of the current node + GenericValue const& value() { + switch (itsType) { + case Value: + return itsValueItBegin[itsIndex]; + case Member: + return itsMemberItBegin[itsIndex].value; + default: + throw cereal::Exception("Invalid Iterator Type!"); } + } - std::string parseObjectToString() - { - rapidjson::StringBuffer sb; - rapidjson::Writer writer( sb ); + //! Get the name of the current node, or nullptr if it has no name + const char* name() const { + if (itsType == Member && (itsMemberItBegin + itsIndex) != itsMemberItEnd) + return itsMemberItBegin[itsIndex].name.GetString(); + else + return nullptr; + } - itsIteratorStack.back().value().Accept( writer ); + //! Adjust our position such that we are at the node with the given name + /*! @throws Exception if no such named node exists */ + inline void search(const char* searchName) //, GenericValue const & parent ) + { + size_t index = 0; + for (auto it = itsMemberItBegin; it != itsMemberItEnd; ++it, ++index) { + if (std::strcmp(searchName, it->name.GetString()) == 0) { + itsIndex = index; + return; + } + } - ++itsIteratorStack.back(); - return sb.GetString(); + if (_optional) { + itsIndex = -1; + return; } - //! @} + throw std::runtime_error("JSON Parsing failed - provided NVP not found"); + } - private: - const char * itsNextName; //!< Next name set by NVP - ReadStream itsReadStream; //!< Rapidjson write stream - std::vector itsIteratorStack; //!< 'Stack' of rapidJSON iterators - rapidjson::Document itsDocument; //!< Rapidjson document + bool found() { return itsIndex != size_t(-1); } + + void setOptional(bool opt) { _optional = opt; } + + private: + MemberIterator itsMemberItBegin, itsMemberItEnd; //!< The member iterator (object) + ValueIterator itsValueItBegin, itsValueItEnd; //!< The value iterator (array) + public: + size_t itsIndex; //!< The current index of this iterator + private: + enum Type { + Value, + Member, + Null + } itsType; //!< Whether this holds values (array) or members (objects) or nothing + bool _optional = false; + }; - bool _foundOptionalValue; + //! Searches for the expectedName node if it doesn't match the actualName + /*! This needs to be called before every load or node start occurs. This function will + check to see if an NVP has been provided (with setNextName) and if so, see if that name matches the actual + next name given. If the names do not match, it will search in the current level of the JSON for that name. + If the name is not found, an exception will be thrown. - public: - unsigned int openJsonNodes = 0; + Resets the NVP name after called. - bool foundOptionalValue() - { - return _foundOptionalValue; - } + @throws Exception if an expectedName is given and not found */ + inline void search() { + // The name an NVP provided with setNextName() + if (itsNextName) { + // The actual name of the current node + auto const actualName = itsIteratorStack.back().name(); - void setOptional(bool opt) - { - itsIteratorStack.back().setOptional(opt); - } - }; + // Do a search if we don't see a name coming up, or if the names don't match + if (!actualName || std::strcmp(itsNextName, actualName) != 0) + itsIteratorStack.back().search(itsNextName); + } - // ###################################################################### - // JSONArchive prologue and epilogue functions - // ###################################################################### - - // ###################################################################### - //! Prologue for NVPs for JSON archives - /*! NVPs do not start or finish nodes - they just set up the names */ - template inline - void prologue( JSONOutputArchive &, NameValuePair const & ) - { } - - //! Prologue for NVPs for JSON archives - template inline - void prologue( JSONInputArchive &, NameValuePair const & ) - { } - - // ###################################################################### - //! Epilogue for NVPs for JSON archives - /*! NVPs do not start or finish nodes - they just set up the names */ - template inline - void epilogue( JSONOutputArchive &, NameValuePair const & ) - { } - - //! Epilogue for NVPs for JSON archives - /*! NVPs do not start or finish nodes - they just set up the names */ - template inline - void epilogue( JSONInputArchive &, NameValuePair const & ) - { } - - // ###################################################################### - //! Prologue for SizeTags for JSON archives - /*! SizeTags are strictly ignored for JSON, they just indicate - that the current node should be made into an array */ - template inline - void prologue( JSONOutputArchive & ar, SizeTag const & ) - { - ar.makeArray(); + itsNextName = nullptr; } - //! Prologue for SizeTags for JSON archives - template inline - void prologue( JSONInputArchive &, SizeTag const & ) - { } - - // ###################################################################### - //! Epilogue for SizeTags for JSON archives - /*! SizeTags are strictly ignored for JSON */ - template inline - void epilogue( JSONOutputArchive &, SizeTag const & ) - { } - - //! Epilogue for SizeTags for JSON archives - template inline - void epilogue( JSONInputArchive &, SizeTag const & ) - { } - - // ###################################################################### - //! Prologue for Json::Value for JSON archives - template inline - typename std::enable_if::value, void>::type - prologue( JSONInputArchive & ar, T const & ) - { - ++ar.openJsonNodes; - ar.startNode(); + public: + //! Starts a new node, going into its proper iterator + /*! This places an iterator for the next node to be parsed onto the iterator stack. If the next + node is an array, this will be a value iterator, otherwise it will be a member iterator. + + By default our strategy is to start with the document root node and then recursively iterate through + all children in the order they show up in the document. + We don't need to know NVPs to do this; we'll just blindly load in the order things appear in. + + If we were given an NVP, we will search for it if it does not match our the name of the next node + that would normally be loaded. This functionality is provided by search(). */ + void startNode() { + search(); + + if (itsIteratorStack.back().value().IsArray()) + itsIteratorStack.emplace_back(itsIteratorStack.back().value().Begin(), itsIteratorStack.back().value().End()); + else + itsIteratorStack.emplace_back(itsIteratorStack.back().value().MemberBegin(), + itsIteratorStack.back().value().MemberEnd()); } - // ###################################################################### - //! Epilogue for Json::Value for JSON archives - template inline - typename std::enable_if::value, void>::type - epilogue( JSONInputArchive & ar, T const & ) - { - while (ar.openJsonNodes > 0) - { - ar.finishNode(); - --ar.openJsonNodes; - } + //! Finishes the most recently started node + void finishNode() { + itsIteratorStack.pop_back(); + ++itsIteratorStack.back(); } - // ###################################################################### - //! Prologue for all other types for JSON archives - /*! Starts a new node, named either automatically or by some NVP, - that may be given data by the type about to be archived */ - template inline - typename std::enable_if::value, void>::type - prologue( JSONOutputArchive & ar, T const & ) - { - ar.startNode(); - } + //! Sets the name for the next node created with startNode + void setNextName(const char* name) { itsNextName = name; } - //! Prologue for all other types for JSON archives - template inline - typename std::enable_if::value && !std::is_same::value, void>::type - prologue( JSONInputArchive & ar, T const & ) - { - ar.startNode(); - } + //! Loads a value from the current node - small signed overload + template + inline typename std::enable_if < std::is_signed::value && + sizeof(T)::type loadValue(T& val) { + search(); - // ###################################################################### - //! Epilogue for all other types other for JSON archives - /*! Finishes the node created in the prologue */ - template inline - typename std::enable_if::value, void>::type - epilogue( JSONOutputArchive & ar, T const & ) - { - ar.finishNode(); + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetInt(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - //! Epilogue for all other types other for JSON archives - template inline - typename std::enable_if::value && !std::is_same::value, void>::type - epilogue( JSONInputArchive & ar, T const & ) - { - ar.finishNode(); + //! Loads a value from the current node - small unsigned overload + template + inline typename std::enable_if< + (std::is_unsigned::value && sizeof(T) < sizeof(uint64_t)) && !std::is_same::value, + void>::type + loadValue(T& val) { + search(); + + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetUint(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - // ###################################################################### - //! Prologue for arithmetic types for JSON archives - template inline - typename std::enable_if::value, void>::type - prologue( JSONOutputArchive & ar, T const & ) - { - ar.writeName(); + //! Loads a value from the current node - bool overload + void loadValue(bool& val) { + search(); + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetBool(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - - //! Prologue for arithmetic types for JSON archives - template inline - typename std::enable_if::value, void>::type - prologue( JSONInputArchive &, T const & ) - { } - - // ###################################################################### - //! Epilogue for arithmetic types for JSON archives - template inline - typename std::enable_if::value, void>::type - epilogue( JSONOutputArchive &, T const & ) - { } - - //! Epilogue for arithmetic types for JSON archives - template inline - typename std::enable_if::value, void>::type - epilogue( JSONInputArchive &, T const & ) - { } - - // ###################################################################### - //! Prologue for strings for JSON archives - template inline - void prologue(JSONOutputArchive & ar, std::basic_string const &) - { - ar.writeName(); + //! Loads a value from the current node - int64 overload + void loadValue(int64_t& val) { + search(); + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetInt64(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - - //! Prologue for strings for JSON archives - template inline - void prologue(JSONInputArchive &, std::basic_string const &) - { } - - // ###################################################################### - //! Epilogue for strings for JSON archives - template inline - void epilogue(JSONOutputArchive &, std::basic_string const &) - { } - - //! Epilogue for strings for JSON archives - template inline - void epilogue(JSONInputArchive &, std::basic_string const &) - { } - - // ###################################################################### - //! Prologue for std::optional<> types for JSON archives - template inline - void prologue(JSONOutputArchive & ar, std::optional const &) - { - ar.writeName(); + //! Loads a value from the current node - uint64 overload + void loadValue(uint64_t& val) { + search(); + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetUint64(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - - //! Prologue for std::optional<> types for JSON archives - template inline - void prologue(JSONInputArchive &, std::optional const &) - { } - - // ###################################################################### - //! Epilogue for std::optional<> types for JSON archives - template inline - void epilogue(JSONOutputArchive &, std::optional const &) - { } - - //! Epilogue for std::optional<> types for JSON archives - template inline - void epilogue(JSONInputArchive &, std::optional const &) - { } - - // ###################################################################### - // Common JSONArchive serialization functions - // ###################################################################### - - //! Serializing NVP types to JSON - template inline - void save( JSONOutputArchive & ar, NameValuePair const & t ) - { - ar.setNextName( t.name ); - ar( t.value ); + //! Loads a value from the current node - float overload + void loadValue(float& val) { + search(); + if (itsIteratorStack.back().found()) { + val = static_cast(itsIteratorStack.back().value().GetDouble()); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - - template inline - void load( JSONInputArchive & ar, NameValuePair & t ) - { - ar.setNextName( t.name ); - ar( t.value ); + //! Loads a value from the current node - double overload + void loadValue(double& val) { + search(); + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetDouble(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - - //! Saving for arithmetic to JSON - template inline - typename std::enable_if::value, void>::type - save(JSONOutputArchive & ar, T const & t) - { - ar.saveValue( t ); + //! Loads a value from the current node - string overload + void loadValue(std::string& val) { + search(); + if (itsIteratorStack.back().found()) { + val = itsIteratorStack.back().value().GetString(); + _foundOptionalValue = true; + } else { + _foundOptionalValue = false; + } + ++itsIteratorStack.back(); } - - //! Loading arithmetic from JSON - template inline - typename std::enable_if::value, void>::type - load(JSONInputArchive & ar, T & t) - { - ar.loadValue( t ); + //! Loads a value from the current node - long double and long long overloads + /*! These data types will automatically be encoded as base64 strings */ + template + inline typename std::enable_if< + std::is_arithmetic::value&&(sizeof(T) >= sizeof(long double) || sizeof(T) >= sizeof(long long)), + void>::type + loadValue(T& val) { + std::string encoded; + loadValue(encoded); + auto decoded = base64::decode(encoded); + + if (sizeof(T) != decoded.size()) + throw Exception("Decoded binary data size does not match specified size"); + + std::memcpy(&val, decoded.data(), decoded.size()); } - //! saving string to JSON - template inline - void save(JSONOutputArchive & ar, std::basic_string const & str) - { - ar.saveValue( str ); - } + //! Loads the size for a SizeTag + void loadSize(size_type& size) { size = (itsIteratorStack.rbegin() + 1)->value().Size(); } - //! loading string from JSON - template inline - void load(JSONInputArchive & ar, std::basic_string & str) - { - ar.loadValue( str ); - } + int valueType() { return (itsIteratorStack.rbegin() + 1)->value().GetType(); } - // ###################################################################### - //! Saving SizeTags to JSON - template inline - void save( JSONOutputArchive &, SizeTag const & ) - { - // nothing to do here, we don't explicitly save the size + std::string parseObjectToString() { + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + + itsIteratorStack.back().value().Accept(writer); + + ++itsIteratorStack.back(); + return sb.GetString(); } - //! Loading SizeTags from JSON - template inline - void load( JSONInputArchive & ar, SizeTag & st ) - { - ar.loadSize( st.size ); + //! @} + + private: + const char* itsNextName; //!< Next name set by NVP + ReadStream itsReadStream; //!< Rapidjson write stream + std::vector itsIteratorStack; //!< 'Stack' of rapidJSON iterators + rapidjson::Document itsDocument; //!< Rapidjson document + + bool _foundOptionalValue; + + public: + unsigned int openJsonNodes = 0; + + bool foundOptionalValue() { return _foundOptionalValue; } + + void setOptional(bool opt) { itsIteratorStack.back().setOptional(opt); } +}; + +// ###################################################################### +// JSONArchive prologue and epilogue functions +// ###################################################################### + +// ###################################################################### +//! Prologue for NVPs for JSON archives +/*! NVPs do not start or finish nodes - they just set up the names */ +template +inline void prologue(JSONOutputArchive&, NameValuePair const&) {} + +//! Prologue for NVPs for JSON archives +template +inline void prologue(JSONInputArchive&, NameValuePair const&) {} + +// ###################################################################### +//! Epilogue for NVPs for JSON archives +/*! NVPs do not start or finish nodes - they just set up the names */ +template +inline void epilogue(JSONOutputArchive&, NameValuePair const&) {} + +//! Epilogue for NVPs for JSON archives +/*! NVPs do not start or finish nodes - they just set up the names */ +template +inline void epilogue(JSONInputArchive&, NameValuePair const&) {} + +// ###################################################################### +//! Prologue for SizeTags for JSON archives +/*! SizeTags are strictly ignored for JSON, they just indicate + that the current node should be made into an array */ +template +inline void prologue(JSONOutputArchive& ar, SizeTag const&) { + ar.makeArray(); +} + +//! Prologue for SizeTags for JSON archives +template +inline void prologue(JSONInputArchive&, SizeTag const&) {} + +// ###################################################################### +//! Epilogue for SizeTags for JSON archives +/*! SizeTags are strictly ignored for JSON */ +template +inline void epilogue(JSONOutputArchive&, SizeTag const&) {} + +//! Epilogue for SizeTags for JSON archives +template +inline void epilogue(JSONInputArchive&, SizeTag const&) {} + +// ###################################################################### +//! Prologue for Json::Value for JSON archives +template +inline typename std::enable_if::value, void>::type prologue(JSONInputArchive& ar, + T const&) { + ++ar.openJsonNodes; + ar.startNode(); +} + +// ###################################################################### +//! Epilogue for Json::Value for JSON archives +template +inline typename std::enable_if::value, void>::type epilogue(JSONInputArchive& ar, + T const&) { + while (ar.openJsonNodes > 0) { + ar.finishNode(); + --ar.openJsonNodes; } -} // namespace cereal +} + +// ###################################################################### +//! Prologue for all other types for JSON archives +/*! Starts a new node, named either automatically or by some NVP, + that may be given data by the type about to be archived */ +template +inline typename std::enable_if::value, void>::type prologue(JSONOutputArchive& ar, T const&) { + ar.startNode(); +} + +//! Prologue for all other types for JSON archives +template +inline typename std::enable_if::value && !std::is_same::value, void>::type +prologue(JSONInputArchive& ar, T const&) { + ar.startNode(); +} + +// ###################################################################### +//! Epilogue for all other types other for JSON archives +/*! Finishes the node created in the prologue */ +template +inline typename std::enable_if::value, void>::type epilogue(JSONOutputArchive& ar, T const&) { + ar.finishNode(); +} + +//! Epilogue for all other types other for JSON archives +template +inline typename std::enable_if::value && !std::is_same::value, void>::type +epilogue(JSONInputArchive& ar, T const&) { + ar.finishNode(); +} + +// ###################################################################### +//! Prologue for arithmetic types for JSON archives +template +inline typename std::enable_if::value, void>::type prologue(JSONOutputArchive& ar, T const&) { + ar.writeName(); +} + +//! Prologue for arithmetic types for JSON archives +template +inline typename std::enable_if::value, void>::type prologue(JSONInputArchive&, T const&) {} + +// ###################################################################### +//! Epilogue for arithmetic types for JSON archives +template +inline typename std::enable_if::value, void>::type epilogue(JSONOutputArchive&, T const&) {} + +//! Epilogue for arithmetic types for JSON archives +template +inline typename std::enable_if::value, void>::type epilogue(JSONInputArchive&, T const&) {} + +// ###################################################################### +//! Prologue for strings for JSON archives +template +inline void prologue(JSONOutputArchive& ar, std::basic_string const&) { + ar.writeName(); +} + +//! Prologue for strings for JSON archives +template +inline void prologue(JSONInputArchive&, std::basic_string const&) {} + +// ###################################################################### +//! Epilogue for strings for JSON archives +template +inline void epilogue(JSONOutputArchive&, std::basic_string const&) {} + +//! Epilogue for strings for JSON archives +template +inline void epilogue(JSONInputArchive&, std::basic_string const&) {} + +// ###################################################################### +//! Prologue for std::optional<> types for JSON archives +template +inline void prologue(JSONOutputArchive& ar, std::optional const&) { + ar.writeName(); +} + +//! Prologue for std::optional<> types for JSON archives +template +inline void prologue(JSONInputArchive&, std::optional const&) {} + +// ###################################################################### +//! Epilogue for std::optional<> types for JSON archives +template +inline void epilogue(JSONOutputArchive&, std::optional const&) {} + +//! Epilogue for std::optional<> types for JSON archives +template +inline void epilogue(JSONInputArchive&, std::optional const&) {} + +// ###################################################################### +// Common JSONArchive serialization functions +// ###################################################################### + +//! Serializing NVP types to JSON +template +inline void save(JSONOutputArchive& ar, NameValuePair const& t) { + ar.setNextName(t.name); + ar(t.value); +} + +template +inline void load(JSONInputArchive& ar, NameValuePair& t) { + ar.setNextName(t.name); + ar(t.value); +} + +//! Saving for arithmetic to JSON +template +inline typename std::enable_if::value, void>::type save(JSONOutputArchive& ar, T const& t) { + ar.saveValue(t); +} + +//! Loading arithmetic from JSON +template +inline typename std::enable_if::value, void>::type load(JSONInputArchive& ar, T& t) { + ar.loadValue(t); +} + +//! saving string to JSON +template +inline void save(JSONOutputArchive& ar, std::basic_string const& str) { + ar.saveValue(str); +} + +//! loading string from JSON +template +inline void load(JSONInputArchive& ar, std::basic_string& str) { + ar.loadValue(str); +} + +// ###################################################################### +//! Saving SizeTags to JSON +template +inline void save(JSONOutputArchive&, SizeTag const&) { + // nothing to do here, we don't explicitly save the size +} + +//! Loading SizeTags from JSON +template +inline void load(JSONInputArchive& ar, SizeTag& st) { + ar.loadSize(st.size); +} +} // namespace cereal // register archives for polymorphic support CEREAL_REGISTER_ARCHIVE(cereal::JSONInputArchive) CEREAL_REGISTER_ARCHIVE(cereal::JSONOutputArchive) -#endif // CEREAL_ARCHIVES_JSON_HPP_ +#endif // CEREAL_ARCHIVES_JSON_HPP_ diff --git a/src/lib/helper/cereal/cereal_type_jsonvalue.h b/src/lib/helper/cereal/cereal_type_jsonvalue.h index 694bba3b0..87289cb7d 100644 --- a/src/lib/helper/cereal/cereal_type_jsonvalue.h +++ b/src/lib/helper/cereal/cereal_type_jsonvalue.h @@ -6,37 +6,33 @@ static const int JSON_VALUE_TYPE_ARRAY = 4; -namespace cereal -{ - //! Loading Json::Value - template inline - void load( Archive & ar, Json::Value & val ) - { - - int valueType = ar.valueType(); - - if (valueType == JSON_VALUE_TYPE_ARRAY) - { - Json::Reader reader; - size_type size; - std::vector values; - std::string str_val; - - ar( make_size_tag( size ) ); - - values.resize( static_cast( size ) ); - for( auto it = values.begin(), end = values.end(); it != end; ++it ) - { - str_val = ar.parseObjectToString(); - reader.parse(str_val, *it); - } - - for (auto jason_value : values) - val.append(jason_value); - } else { - std::cerr << "Error: Unable to serialize given type into Json::Value." << std::endl; +namespace cereal { +//! Loading Json::Value +template +inline void load(Archive& ar, Json::Value& val) { + + int valueType = ar.valueType(); + + if (valueType == JSON_VALUE_TYPE_ARRAY) { + Json::Reader reader; + size_type size; + std::vector values; + std::string str_val; + + ar(make_size_tag(size)); + + values.resize(static_cast(size)); + for (auto it = values.begin(), end = values.end(); it != end; ++it) { + str_val = ar.parseObjectToString(); + reader.parse(str_val, *it); } + + for (auto jason_value : values) + val.append(jason_value); + } else { + std::cerr << "Error: Unable to serialize given type into Json::Value." << std::endl; } -} // namespace cereal +} +} // namespace cereal -#endif // CEREAL_TYPES_JSONVALUE_H_ +#endif // CEREAL_TYPES_JSONVALUE_H_ diff --git a/src/lib/helper/cereal/cereal_type_optional.h b/src/lib/helper/cereal/cereal_type_optional.h index 7a1b51dd5..6e9ae1340 100644 --- a/src/lib/helper/cereal/cereal_type_optional.h +++ b/src/lib/helper/cereal/cereal_type_optional.h @@ -3,19 +3,17 @@ #include -namespace cereal -{ - //! Loading std::optional - template inline - void load( Archive & ar, std::optional & opt ) - { - ar.setOptional(true); - T optValue; - ar( optValue ); - if (ar.foundOptionalValue()) opt = std::optional(optValue); - ar.setOptional(false); - } -} // namespace cereal - -#endif // CEREAL_TYPES_OPTIONAL_H_ +namespace cereal { +//! Loading std::optional +template +inline void load(Archive& ar, std::optional& opt) { + ar.setOptional(true); + T optValue; + ar(optValue); + if (ar.foundOptionalValue()) + opt = std::optional(optValue); + ar.setOptional(false); +} +} // namespace cereal +#endif // CEREAL_TYPES_OPTIONAL_H_ diff --git a/src/lib/helper/cereal/serialize_macro.h b/src/lib/helper/cereal/serialize_macro.h index 7853ffc86..fb5359e67 100644 --- a/src/lib/helper/cereal/serialize_macro.h +++ b/src/lib/helper/cereal/serialize_macro.h @@ -2,63 +2,263 @@ #define SERIALIZE_MACRO_H_ -#define CEREAL(...) template\ - void serialize( Archive & archive )\ - {\ - archive(\ - __VA_ARGS__\ - );\ - } +#define CEREAL(...) \ + template \ + void serialize(Archive& archive) { \ + archive(__VA_ARGS__); \ + } -#define SERIALIZE_0() -#define SERIALIZE_1(A) CEREAL(cereal::make_nvp(#A, A)) -#define SERIALIZE_2(A,B) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B)) -#define SERIALIZE_3(A,B,C) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C)) -#define SERIALIZE_4(A,B,C,D) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D)) -#define SERIALIZE_5(A,B,C,D,E) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E)) -#define SERIALIZE_6(A,B,C,D,E,F) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F)) -#define SERIALIZE_7(A,B,C,D,E,F,G) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G)) -#define SERIALIZE_8(A,B,C,D,E,F,G,H) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H)) -#define SERIALIZE_9(A,B,C,D,E,F,G,H,I) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I)) -#define SERIALIZE_10(A,B,C,D,E,F,G,H,I,J) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J)) -#define SERIALIZE_11(A,B,C,D,E,F,G,H,I,J,K) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K)) -#define SERIALIZE_12(A,B,C,D,E,F,G,H,I,J,K,L) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L)) -#define SERIALIZE_13(A,B,C,D,E,F,G,H,I,J,K,L,M) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M)) -#define SERIALIZE_14(A,B,C,D,E,F,G,H,I,J,K,L,M,N) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N)) -#define SERIALIZE_15(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O)) -#define SERIALIZE_16(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P)) -#define SERIALIZE_17(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q)) -#define SERIALIZE_18(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q), cereal::make_nvp(#R, R)) -#define SERIALIZE_19(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q), cereal::make_nvp(#R, R), cereal::make_nvp(#S, S)) -#define SERIALIZE_20(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D), cereal::make_nvp(#E, E), cereal::make_nvp(#F, F), cereal::make_nvp(#G, G), cereal::make_nvp(#H, H), cereal::make_nvp(#I, I), cereal::make_nvp(#J, J), cereal::make_nvp(#K, K), cereal::make_nvp(#L, L), cereal::make_nvp(#M, M), cereal::make_nvp(#N, N), cereal::make_nvp(#O, O), cereal::make_nvp(#P, P), cereal::make_nvp(#Q, Q), cereal::make_nvp(#R, R), cereal::make_nvp(#S, S), cereal::make_nvp(#T, T)) +#define SERIALIZE_0() +#define SERIALIZE_1(A) CEREAL(cereal::make_nvp(#A, A)) +#define SERIALIZE_2(A, B) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B)) +#define SERIALIZE_3(A, B, C) CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C)) +#define SERIALIZE_4(A, B, C, D) \ + CEREAL(cereal::make_nvp(#A, A), cereal::make_nvp(#B, B), cereal::make_nvp(#C, C), cereal::make_nvp(#D, D)) +#define SERIALIZE_5(A, B, C, D, E) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E)) +#define SERIALIZE_6(A, B, C, D, E, F) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F)) +#define SERIALIZE_7(A, B, C, D, E, F, G) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G)) +#define SERIALIZE_8(A, B, C, D, E, F, G, H) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H)) +#define SERIALIZE_9(A, B, C, D, E, F, G, H, I) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I)) +#define SERIALIZE_10(A, B, C, D, E, F, G, H, I, J) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J)) +#define SERIALIZE_11(A, B, C, D, E, F, G, H, I, J, K) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K)) +#define SERIALIZE_12(A, B, C, D, E, F, G, H, I, J, K, L) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L)) +#define SERIALIZE_13(A, B, C, D, E, F, G, H, I, J, K, L, M) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M)) +#define SERIALIZE_14(A, B, C, D, E, F, G, H, I, J, K, L, M, N) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N)) +#define SERIALIZE_15(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N), \ + cereal::make_nvp(#O, O)) +#define SERIALIZE_16(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N), \ + cereal::make_nvp(#O, O), \ + cereal::make_nvp(#P, P)) +#define SERIALIZE_17(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N), \ + cereal::make_nvp(#O, O), \ + cereal::make_nvp(#P, P), \ + cereal::make_nvp(#Q, Q)) +#define SERIALIZE_18(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N), \ + cereal::make_nvp(#O, O), \ + cereal::make_nvp(#P, P), \ + cereal::make_nvp(#Q, Q), \ + cereal::make_nvp(#R, R)) +#define SERIALIZE_19(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N), \ + cereal::make_nvp(#O, O), \ + cereal::make_nvp(#P, P), \ + cereal::make_nvp(#Q, Q), \ + cereal::make_nvp(#R, R), \ + cereal::make_nvp(#S, S)) +#define SERIALIZE_20(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) \ + CEREAL(cereal::make_nvp(#A, A), \ + cereal::make_nvp(#B, B), \ + cereal::make_nvp(#C, C), \ + cereal::make_nvp(#D, D), \ + cereal::make_nvp(#E, E), \ + cereal::make_nvp(#F, F), \ + cereal::make_nvp(#G, G), \ + cereal::make_nvp(#H, H), \ + cereal::make_nvp(#I, I), \ + cereal::make_nvp(#J, J), \ + cereal::make_nvp(#K, K), \ + cereal::make_nvp(#L, L), \ + cereal::make_nvp(#M, M), \ + cereal::make_nvp(#N, N), \ + cereal::make_nvp(#O, O), \ + cereal::make_nvp(#P, P), \ + cereal::make_nvp(#Q, Q), \ + cereal::make_nvp(#R, R), \ + cereal::make_nvp(#S, S), \ + cereal::make_nvp(#T, T)) // The interim macro that simply strips the excess and ends up with the required macro -#define SERIALIZE_X(x,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,FUNC, ...) FUNC +#define SERIALIZE_X(x, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, FUNC, ...) FUNC -// The macro that the programmer uses -#define SERIALIZE(...) SERIALIZE_X(,##__VA_ARGS__,\ - SERIALIZE_20(__VA_ARGS__),\ - SERIALIZE_19(__VA_ARGS__),\ - SERIALIZE_18(__VA_ARGS__),\ - SERIALIZE_17(__VA_ARGS__),\ - SERIALIZE_16(__VA_ARGS__),\ - SERIALIZE_15(__VA_ARGS__),\ - SERIALIZE_14(__VA_ARGS__),\ - SERIALIZE_13(__VA_ARGS__),\ - SERIALIZE_12(__VA_ARGS__),\ - SERIALIZE_11(__VA_ARGS__),\ - SERIALIZE_10(__VA_ARGS__),\ - SERIALIZE_9(__VA_ARGS__),\ - SERIALIZE_8(__VA_ARGS__),\ - SERIALIZE_7(__VA_ARGS__),\ - SERIALIZE_6(__VA_ARGS__),\ - SERIALIZE_5(__VA_ARGS__),\ - SERIALIZE_4(__VA_ARGS__),\ - SERIALIZE_3(__VA_ARGS__),\ - SERIALIZE_2(__VA_ARGS__),\ - SERIALIZE_1(__VA_ARGS__),\ - SERIALIZE_0(__VA_ARGS__)\ - ) +// The macro that the programmer uses +#define SERIALIZE(...) \ + SERIALIZE_X(, \ + ##__VA_ARGS__, \ + SERIALIZE_20(__VA_ARGS__), \ + SERIALIZE_19(__VA_ARGS__), \ + SERIALIZE_18(__VA_ARGS__), \ + SERIALIZE_17(__VA_ARGS__), \ + SERIALIZE_16(__VA_ARGS__), \ + SERIALIZE_15(__VA_ARGS__), \ + SERIALIZE_14(__VA_ARGS__), \ + SERIALIZE_13(__VA_ARGS__), \ + SERIALIZE_12(__VA_ARGS__), \ + SERIALIZE_11(__VA_ARGS__), \ + SERIALIZE_10(__VA_ARGS__), \ + SERIALIZE_9(__VA_ARGS__), \ + SERIALIZE_8(__VA_ARGS__), \ + SERIALIZE_7(__VA_ARGS__), \ + SERIALIZE_6(__VA_ARGS__), \ + SERIALIZE_5(__VA_ARGS__), \ + SERIALIZE_4(__VA_ARGS__), \ + SERIALIZE_3(__VA_ARGS__), \ + SERIALIZE_2(__VA_ARGS__), \ + SERIALIZE_1(__VA_ARGS__), \ + SERIALIZE_0(__VA_ARGS__)) -#endif // SERIALIZE_MACRO_H_ +#endif // SERIALIZE_MACRO_H_ From 1c9248068615f2e8bb175d9dc00d573356c007fb Mon Sep 17 00:00:00 2001 From: lanice Date: Tue, 8 Apr 2014 14:17:25 +0000 Subject: [PATCH 4/5] Replace Json::Value::toStyledString() with Json::FastWriter::write() in Queryparser.h --- src/lib/access/system/QueryParser.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/access/system/QueryParser.h b/src/lib/access/system/QueryParser.h index 073b0e862..fde727cb5 100644 --- a/src/lib/access/system/QueryParser.h +++ b/src/lib/access/system/QueryParser.h @@ -64,7 +64,8 @@ struct QueryParserFactory : public AbstractQueryParserFacto virtual std::shared_ptr parse(const Json::Value& data) { std::stringstream ss; - ss << data.toStyledString(); + Json::FastWriter writer; + ss << writer.write(data); cereal::JSONInputArchive archive(ss); From fe50ffffa108e07758e5a8e5fcbcd2081911da14 Mon Sep 17 00:00:00 2001 From: lanice Date: Tue, 8 Apr 2014 15:50:48 +0000 Subject: [PATCH 5/5] replace doubled parameters with Parameters struct in TableLoad and SimpleTableScan --- src/lib/access/SimpleTableScan.cpp | 12 ++-- src/lib/access/SimpleTableScan.h | 6 +- src/lib/access/storage/TableLoad.cpp | 93 ++++++++++++---------------- src/lib/access/storage/TableLoad.h | 16 +---- 4 files changed, 49 insertions(+), 78 deletions(-) diff --git a/src/lib/access/SimpleTableScan.cpp b/src/lib/access/SimpleTableScan.cpp index cfadb4733..e509f3484 100644 --- a/src/lib/access/SimpleTableScan.cpp +++ b/src/lib/access/SimpleTableScan.cpp @@ -19,14 +19,14 @@ namespace { auto _ = QueryParser::registerSerializablePlanOperation("SimpleTableScan"); } -SimpleTableScan::SimpleTableScan() : _comparator(nullptr), _ofDelta(false) {} - -SimpleTableScan::SimpleTableScan(const Parameters& parameters) : _comparator(nullptr), _ofDelta(false) { +SimpleTableScan::SimpleTableScan(const Parameters& parameters) : _comparator(nullptr) { setPredicate(buildExpression(parameters.predicates)); if (parameters.materializing) setProducesPositions(!*parameters.materializing); if (parameters.ofDelta) - _ofDelta = *parameters.ofDelta; + _parameters.ofDelta = *parameters.ofDelta; + else + _parameters.ofDelta = false; } SimpleTableScan::~SimpleTableScan() { @@ -41,7 +41,7 @@ void SimpleTableScan::executePositional() { storage::pos_list_t* pos_list = new pos_list_t(); - size_t row = _ofDelta ? checked_pointer_cast(tbl)->deltaOffset() : 0; + size_t row = *_parameters.ofDelta ? checked_pointer_cast(tbl)->deltaOffset() : 0; for (size_t input_size = tbl->size(); row < input_size; ++row) { if ((*_comparator)(row)) { pos_list->push_back(row); @@ -55,7 +55,7 @@ void SimpleTableScan::executeMaterialized() { auto result_table = tbl->copy_structure_modifiable(); size_t target_row = 0; - size_t row = _ofDelta ? checked_pointer_cast(tbl)->deltaOffset() : 0; + size_t row = *_parameters.ofDelta ? checked_pointer_cast(tbl)->deltaOffset() : 0; for (size_t input_size = tbl->size(); row < input_size; ++row) { if ((*_comparator)(row)) { // TODO materializing result set will make the allocation the boundary diff --git a/src/lib/access/SimpleTableScan.h b/src/lib/access/SimpleTableScan.h index 3f2518476..27fb64e76 100644 --- a/src/lib/access/SimpleTableScan.h +++ b/src/lib/access/SimpleTableScan.h @@ -21,8 +21,7 @@ class SimpleTableScan : public ParallelizablePlanOperation { }; public: - SimpleTableScan(); - SimpleTableScan(const Parameters& parameters); + SimpleTableScan(const Parameters& parameters = Parameters()); virtual ~SimpleTableScan(); void setupPlanOperation(); @@ -34,8 +33,9 @@ class SimpleTableScan : public ParallelizablePlanOperation { void setPredicate(SimpleExpression* c); private: + Parameters _parameters; + SimpleExpression* _comparator; - bool _ofDelta; }; } } diff --git a/src/lib/access/storage/TableLoad.cpp b/src/lib/access/storage/TableLoad.cpp index 32cc75f45..0a111bdac 100644 --- a/src/lib/access/storage/TableLoad.cpp +++ b/src/lib/access/storage/TableLoad.cpp @@ -21,101 +21,84 @@ log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("access.plan.PlanOperation" } -TableLoad::TableLoad(const Parameters& parameters) - : _table_name(parameters.table), - _file_name(parameters.filename), - _header_file_name(parameters.header), - _header_string(parameters.header_string), - _delimiter(parameters.delimiter), - _path(parameters.path), - _unsafe(parameters.unsafe), - _raw(parameters.raw) { - if (parameters.path) - _path = *parameters.path; - else - _path = std::string(""); - if (parameters.unsafe) - _unsafe = *parameters.unsafe; - else - _unsafe = false; - if (parameters.raw) - _raw = *parameters.raw; - else - _raw = false; +TableLoad::TableLoad(const Parameters& parameters) : _parameters(parameters) { + if (!_parameters.path) + _parameters.path = std::string(""); + if (!_parameters.unsafe) + _parameters.unsafe = false; + if (!_parameters.raw) + _parameters.raw = false; } -TableLoad::TableLoad() : _path(std::string("")), _unsafe(false), _raw(false), _nonvolatile(false), _binary(false) {} - TableLoad::~TableLoad() {} void TableLoad::executePlanOperation() { auto sm = io::StorageManager::getInstance(); - if (!sm->exists(_table_name)) { + if (!sm->exists(_parameters.table)) { // load from absolute path? // Load Raw Table - if (*_raw) { + if (*_parameters.raw) { io::Loader::params p; - p.setHeader(io::CSVHeader(_file_name)); - p.setInput(io::RawTableLoader(_file_name)); - sm->loadTable(_table_name, p, *_path); + p.setHeader(io::CSVHeader(_parameters.filename)); + p.setInput(io::RawTableLoader(_parameters.filename)); + sm->loadTable(_parameters.table, p, *_parameters.path); - } else if (_header_string) { + } else if (_parameters.header_string) { // Load based on header string - auto p = io::Loader::shortcuts::loadWithStringHeaderParams(_file_name, *_header_string); - sm->loadTable(_table_name, p, *_path); + auto p = io::Loader::shortcuts::loadWithStringHeaderParams(_parameters.filename, *_parameters.header_string); + sm->loadTable(_parameters.table, p, *_parameters.path); - } else if (!_header_file_name) { + } else if (!_parameters.header) { // Load only with single file - sm->loadTableFile(_table_name, _file_name, *_path); + sm->loadTableFile(_parameters.table, _parameters.filename, *_parameters.path); - } else if ((!_table_name.empty()) && (!_file_name.empty()) && (_header_file_name)) { + } else if ((!_parameters.table.empty()) && (!_parameters.filename.empty()) && (_parameters.header)) { // Load with dedicated header file io::Loader::params p; p.setCompressed(false); - p.setHeader(io::CSVHeader(*_header_file_name)); - auto params = io::CSVInput::params().setUnsafe(*_unsafe); - if (_delimiter) - params.setCSVParams(io::csv::params().setDelimiter((*_delimiter).at(0))); - p.setInput(io::CSVInput(_file_name, params)); - sm->loadTable(_table_name, p, *_path); + p.setHeader(io::CSVHeader(*_parameters.header)); + auto params = io::CSVInput::params().setUnsafe(*_parameters.unsafe); + if (_parameters.delimiter) + params.setCSVParams(io::csv::params().setDelimiter((*_parameters.delimiter).at(0))); + p.setInput(io::CSVInput(_parameters.filename, params)); + sm->loadTable(_parameters.table, p, *_parameters.path); } - auto table = sm->getTable(_table_name); - table->setName(_table_name); + auto table = sm->getTable(_parameters.table); + table->setName(_parameters.table); // We don't load unless the necessary prerequisites are met, // let StorageManager error if table does not exist } else { - sm->getTable(_table_name); + sm->getTable(_parameters.table); } - auto _table = sm->getTable(_table_name); + auto _table = sm->getTable(_parameters.table); LOG4CXX_DEBUG(logger, "Loaded Table Size" << _table->size()); addResult(_table); } const std::string TableLoad::vname() { return "TableLoad"; } -void TableLoad::setTableName(const std::string& tablename) { _table_name = tablename; } +void TableLoad::setTableName(const std::string& tablename) { _parameters.table = tablename; } -void TableLoad::setFileName(const std::string& filename) { _file_name = filename; } +void TableLoad::setFileName(const std::string& filename) { _parameters.filename = filename; } void TableLoad::setHeaderFileName(const std::string& filename) { - _header_file_name = std::optional(filename); + _parameters.header = std::optional(filename); } -void TableLoad::setHeaderString(const std::string& header) { _header_string = std::optional(header); } - -void TableLoad::setUnsafe(const bool unsafe) { _unsafe = std::optional(unsafe); } +void TableLoad::setHeaderString(const std::string& header) { + _parameters.header_string = std::optional(header); +} -void TableLoad::setRaw(const bool raw) { _raw = std::optional(raw); } +void TableLoad::setUnsafe(const bool unsafe) { _parameters.unsafe = std::optional(unsafe); } -void TableLoad::setDelimiter(const std::string& d) { _delimiter = std::optional(d); } -void TableLoad::setPath(const std::string& path) { _path = std::optional(path); } +void TableLoad::setRaw(const bool raw) { _parameters.raw = std::optional(raw); } -void TableLoad::setBinary(const bool binary) { _binary = binary; } +void TableLoad::setDelimiter(const std::string& d) { _parameters.delimiter = std::optional(d); } -void TableLoad::setNonvolatile(const bool nonvolatile) { _nonvolatile = nonvolatile; } +void TableLoad::setPath(const std::string& path) { _parameters.path = std::optional(path); } } } diff --git a/src/lib/access/storage/TableLoad.h b/src/lib/access/storage/TableLoad.h index 9f6271195..72a930520 100644 --- a/src/lib/access/storage/TableLoad.h +++ b/src/lib/access/storage/TableLoad.h @@ -25,8 +25,7 @@ class TableLoad : public PlanOperation { }; public: - TableLoad(); - TableLoad(const Parameters& parameters); + TableLoad(const Parameters& parameters = Parameters()); virtual ~TableLoad(); void executePlanOperation(); @@ -37,23 +36,12 @@ class TableLoad : public PlanOperation { void setPath(const std::string& path); void setHeaderFileName(const std::string& filename); void setHeaderString(const std::string& header); - void setBinary(const bool binary); void setUnsafe(const bool unsafe); void setRaw(const bool raw); void setDelimiter(const std::string& d); - void setNonvolatile(const bool nonvolatile); private: - std::string _table_name; - std::string _file_name; - std::optional _header_file_name; - std::optional _header_string; - std::optional _delimiter; - std::optional _path; - std::optional _unsafe; - std::optional _raw; - bool _nonvolatile; - bool _binary; + Parameters _parameters; }; } }