diff --git a/modules/dmrpp_module/DMZ.cc b/modules/dmrpp_module/DMZ.cc index 8f75839d4d..e9d5d91c7a 100644 --- a/modules/dmrpp_module/DMZ.cc +++ b/modules/dmrpp_module/DMZ.cc @@ -92,7 +92,6 @@ using namespace libdap; namespace dmrpp { - using shape = std::vector; // The original unsupported fillValue flags from 4/22 @@ -108,16 +107,16 @@ bool DMZ::d_elide_unsupported = true; #if 1 -const std::set DMZ::variable_elements{"Byte", "Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", - "UInt64", "Float32", "Float64", "String", "Structure", "Sequence", - "Enum", "Opaque"}; +const std::set DMZ::variable_elements{ + "Byte", "Int8", "Int16", "Int32", "Int64", "UInt8", "UInt16", "UInt32", + "UInt64", "Float32", "Float64", "String", "Structure", "Sequence", + "Enum", "Opaque" +}; #endif - /// @brief Are the C-style strings equal? -static inline bool is_eq(const char *value, const char *key) -{ +static inline bool is_eq(const char *value, const char *key) { #if TREAT_NAMESPACES_AS_LITERALS return strcmp(value, key) == 0; #else @@ -125,31 +124,28 @@ static inline bool is_eq(const char *value, const char *key) return true; } else { - const char* colon = strchr(value, ':'); + const char *colon = strchr(value, ':'); return colon && strcmp(colon + 1, key) == 0; } #endif } /// @brief Are any of the child nodes 'Dim' elements? -static inline bool has_dim_nodes(const xml_node &var_node) -{ +static inline bool has_dim_nodes(const xml_node &var_node) { return var_node.child("Dim"); // just one is enough } /// @brief Simple set membership; used to test for variable elements, et cetera. -static inline bool member_of(const set &elements_set, const string &element_name) -{ +static inline bool member_of(const set &elements_set, const string &element_name) { return elements_set.find(element_name) != elements_set.end(); } /// @brief syntactic sugar for a dynamic cast to DmrppCommon -static inline DmrppCommon *dc(BaseType *btp) -{ - auto *dc = dynamic_cast(btp); +static inline DmrppCommon *dc(BaseType *btp) { + auto *dc = dynamic_cast(btp); if (!dc) throw BESInternalError(string("Expected a BaseType that was also a DmrppCommon instance (") - .append((btp) ? btp->name() : "unknown").append(")."), __FILE__, __LINE__); + .append((btp) ? btp->name() : "unknown").append(")."), __FILE__, __LINE__); return dc; } @@ -158,12 +154,11 @@ static inline DmrppCommon *dc(BaseType *btp) * Loads configuration state from TheBESKeys * */ -void DMZ::load_config_from_keys() -{ +void DMZ::load_config_from_keys() { // ######################################################################## - // Loads the ELIDE_UNSUPPORTED_KEY (see top of file for key definition) + // Loads the ELIDE_UNSUPPORTED_KEY (see top of this file for key definition) // And if it's set, and set to true, then we set the eliding flag to true. - d_elide_unsupported = TheBESKeys::TheKeys()->read_bool_key(ELIDE_UNSUPPORTED_KEY,false); + d_elide_unsupported = TheBESKeys::read_bool_key(ELIDE_UNSUPPORTED_KEY, false); } /** @@ -171,8 +166,7 @@ void DMZ::load_config_from_keys() * @param file_name The DMR++ XML document to parse. * @exception BESInternalError if file_name cannot be parsed */ -DMZ::DMZ(const string &file_name) -{ +DMZ::DMZ(const string &file_name) { load_config_from_keys(); parse_xml_doc(file_name); } @@ -182,8 +176,7 @@ DMZ::DMZ(const string &file_name) * @param file_name */ void -DMZ::parse_xml_doc(const string &file_name) -{ +DMZ::parse_xml_doc(const string &file_name) { std::ifstream stream(file_name); // Free memory used by a previously parsed document. @@ -191,7 +184,7 @@ DMZ::parse_xml_doc(const string &file_name) // parse_ws_pcdata_single will include the space when it appears in a // DAP Attribute element. jhrg 11/3/21 - pugi::xml_parse_result result = d_xml_doc.load(stream, pugi::parse_default | pugi::parse_ws_pcdata_single); + pugi::xml_parse_result result = d_xml_doc.load(stream, pugi::parse_default | pugi::parse_ws_pcdata_single); if (!result) throw BESInternalError(string("DMR++ parse error: ").append(result.description()), __FILE__, __LINE__); @@ -200,8 +193,6 @@ DMZ::parse_xml_doc(const string &file_name) throw BESInternalError("No DMR++ data present.", __FILE__, __LINE__); } - - /** * * @param var_node @@ -218,22 +209,22 @@ bool flagged_as_unsupported_type(xml_node var_node, string &unsupported_flag) { // We know the unsupported flag is held in the fillValue attribute of the dmrpp:chunks element. auto chunks = var_node.child("dmrpp:chunks"); - if(!chunks) { + if (!chunks) { // No dmrpp:chunks? Then no fillValue and we can be done, it's supported. return is_unsupported_type; } - xml_attribute fillValue_attr = chunks.attribute("fillValue"); - if(!fillValue_attr) { + xml_attribute fillValue_attr = chunks.attribute("fillValue"); + if (!fillValue_attr) { // No fillValue attribute? Then we can be done, it's supported. return is_unsupported_type; } // We found th fillValue attribute, So now we have to deal with its various tragic values... - if(is_eq(fillValue_attr.value(), UNSUPPORTED_STRING)){ + if (is_eq(fillValue_attr.value(), UNSUPPORTED_STRING)) { // UNSUPPORTED_STRING is the older, indeterminate, tag which might label a truly - // unsupported VariableLengthString or it could be a labeling FixedLengthString. - // In order to find out we need to look in XML DOM to determine if this is an Array, and + // unsupported VariableLengthString, or it could be a labeling FixedLengthString. + // To find out, we need to look in XML DOM to determine if this is an Array, and // if so, to see if it's the FixedLengthString case: // // This should be a child of var_node. @@ -242,8 +233,8 @@ bool flagged_as_unsupported_type(xml_node var_node, string &unsupported_flag) { is_unsupported_type = true; auto dim_node = var_node.child("Dim"); - if(!dim_node) { - // No dims? Then this is a scalar String and it's cool. + if (!dim_node) { + // No dims? Then this is a scalar String, and it's cool. // We dump the BS fillValue for one that makes some sense in Stringville fillValue_attr.set_value(""); is_unsupported_type = false; @@ -251,7 +242,7 @@ bool flagged_as_unsupported_type(xml_node var_node, string &unsupported_flag) { else { // It's an array, so is it a FixedLengthStringArray?? auto flsa_node = var_node.child("dmrpp:FixedLengthStringArray"); - if(flsa_node){ + if (flsa_node) { // FixedLengthStringArray arrays work! // We dump the BS fillValue for one that makes some sense in Stringville fillValue_attr.set_value(""); @@ -259,24 +250,23 @@ bool flagged_as_unsupported_type(xml_node var_node, string &unsupported_flag) { } } } - else if(is_eq(fillValue_attr.value(),UNSUPPORTED_VARIABLE_LENGTH_STRING)) { - unsupported_flag=fillValue_attr.value(); + else if (is_eq(fillValue_attr.value(), UNSUPPORTED_VARIABLE_LENGTH_STRING)) { + unsupported_flag = fillValue_attr.value(); is_unsupported_type = true; } - else if(is_eq(fillValue_attr.value(),UNSUPPORTED_ARRAY)){ - unsupported_flag=fillValue_attr.value(); - is_unsupported_type = true; + else if (is_eq(fillValue_attr.value(), UNSUPPORTED_ARRAY)) { + unsupported_flag = fillValue_attr.value(); + is_unsupported_type = true; } - else if(is_eq(fillValue_attr.value(),UNSUPPORTED_COMPOUND)){ - unsupported_flag=fillValue_attr.value(); - is_unsupported_type = true; + else if (is_eq(fillValue_attr.value(), UNSUPPORTED_COMPOUND)) { + unsupported_flag = fillValue_attr.value(); + is_unsupported_type = true; } return is_unsupported_type; } - /** * @brief Build a DOM tree for a DMR++ using content from a string. * @note This method was added to support parsing DMR++ 'documents' @@ -287,8 +277,7 @@ bool flagged_as_unsupported_type(xml_node var_node, string &unsupported_flag) { * @param source The string that contains the DMR++ content. */ void -DMZ::parse_xml_string(const string &source) -{ +DMZ::parse_xml_string(const string &source) { pugi::xml_parse_result result = d_xml_doc.load_string(source.c_str()); if (!result) @@ -307,13 +296,12 @@ DMZ::parse_xml_string(const string &source) * @param dmr Dump the information in this instance of DMR * @param xml_root The root node of the DOM tree */ -void DMZ::process_dataset(DMR *dmr, const xml_node &xml_root) -{ +void DMZ::process_dataset(DMR *dmr, const xml_node &xml_root) { // Process the attributes - int required_attrs_found = 0; // there are 1 + int required_attrs_found = 0; // there are 1 string href_attr; bool href_trusted = false; - string dmrpp_version; // empty or holds a value if dmrpp::version is present + string dmrpp_version; // empty or holds a value if dmrpp:version is present for (xml_attribute attr = xml_root.first_attribute(); attr; attr = attr.next_attribute()) { if (is_eq(attr.name(), "name")) { ++required_attrs_found; @@ -344,23 +332,25 @@ void DMZ::process_dataset(DMR *dmr, const xml_node &xml_root) else if (is_eq(attr.name(), "dmrpp:version")) { dmrpp_version = attr.value(); } - // We allow other, non recognized attributes, so there is no 'else' jhrg 10/20/21 + // We allow other, unrecognized attributes, so there is no 'else' jhrg 10/20/21 } - if (dmrpp_version.empty()) { // old style DMR++, set enable-kludge flag + if (dmrpp_version.empty()) { + // old style DMR++, set enable-kludge flag DmrppRequestHandler::d_emulate_original_filter_order_behavior = true; } else { - auto dmrpp = dynamic_cast(dmr); + auto dmrpp = dynamic_cast(dmr); if (dmrpp) { dmrpp->set_version(dmrpp_version); } } if (required_attrs_found != 1) - throw BESInternalError("DMR++ XML dataset element missing one or more required attributes.", __FILE__, __LINE__); + throw BESInternalError("DMR++ XML dataset element missing one or more required attributes.", __FILE__, + __LINE__); - if (href_attr.empty()) + if (href_attr.empty()) throw BESInternalError("DMR++ XML dataset element dmrpp:href is missing. ", __FILE__, __LINE__); d_dataset_elem_href.reset(new http::url(href_attr, href_trusted)); @@ -371,8 +361,7 @@ void DMZ::process_dataset(DMR *dmr, const xml_node &xml_root) * @param grp The group we are currently processing (could be the root group) * @param dimension_node The node in the DOM tree of the element */ -void DMZ::process_dimension(D4Group *grp, const xml_node &dimension_node) -{ +void DMZ::process_dimension(D4Group *grp, const xml_node &dimension_node) { string name_value; string size_value; for (xml_attribute attr = dimension_node.first_attribute(); attr; attr = attr.next_attribute()) { @@ -385,7 +374,8 @@ void DMZ::process_dimension(D4Group *grp, const xml_node &dimension_node) } if (name_value.empty() || size_value.empty()) - throw BESInternalError("The required attribute 'name' or 'size' was missing from a Dimension element.", __FILE__, __LINE__); + throw BESInternalError("The required attribute 'name' or 'size' was missing from a Dimension element.", + __FILE__, __LINE__); // This getter (dim_def) allocates a new object if needed. try { @@ -406,8 +396,7 @@ void DMZ::process_dimension(D4Group *grp, const xml_node &dimension_node) * @param array The variable we are processing * @param dim_node The node in the DOM tree of the Dim element */ -void DMZ::process_dim(DMR *dmr, D4Group *grp, Array *array, const xml_node &dim_node) -{ +void DMZ::process_dim(DMR *dmr, D4Group *grp, Array *array, const xml_node &dim_node) { string name_value; string size_value; for (xml_attribute attr = dim_node.first_attribute(); attr; attr = attr.next_attribute()) { @@ -422,7 +411,8 @@ void DMZ::process_dim(DMR *dmr, D4Group *grp, Array *array, const xml_node &dim_ if (name_value.empty() && size_value.empty()) throw BESInternalError("Either 'size' or 'name' must be used in a Dim element.", __FILE__, __LINE__); if (!name_value.empty() && !size_value.empty()) - throw BESInternalError("Only one of 'size' and 'name' are allowed in a Dim element, but both were used.", __FILE__, __LINE__); + throw BESInternalError("Only one of 'size' and 'name' are allowed in a Dim element, but both were used.", + __FILE__, __LINE__); if (!size_value.empty()) { BESDEBUG(PARSER, prolog << "Processing nameless Dim of size: " << stoll(size_value) << endl); @@ -432,21 +422,22 @@ void DMZ::process_dim(DMR *dmr, D4Group *grp, Array *array, const xml_node &dim_ BESDEBUG(PARSER, prolog << "Processing Dim with named Dimension reference: " << name_value << endl); D4Dimension *dim; - if (name_value[0] == '/') // lookup the Dimension in the root group + if (name_value[0] == '/') // lookup the Dimension in the root group dim = dmr->root()->find_dim(name_value); else // get enclosing Group and lookup Dimension there dim = grp->find_dim(name_value); if (!dim) - throw BESInternalError("The dimension '" + name_value + "' was not found while parsing the variable '" + array->name() + "'.",__FILE__,__LINE__); + throw BESInternalError( + "The dimension '" + name_value + "' was not found while parsing the variable '" + array->name() + "'.", + __FILE__,__LINE__); array->append_dim(dim); } } -void DMZ::process_map(DMR *dmr, D4Group *grp, Array *array, const xml_node &map_node) -{ +void DMZ::process_map(DMR *dmr, D4Group *grp, Array *array, const xml_node &map_node) { string name_value; string size_value; for (xml_attribute attr = map_node.first_attribute(); attr; attr = attr.next_attribute()) { @@ -491,20 +482,21 @@ void DMZ::process_map(DMR *dmr, D4Group *grp, Array *array, const xml_node &map_ * @param parent If not null add this new variable to this constructor * @param var_node */ -void DMZ::process_variable(DMR *dmr, D4Group *group, Constructor *parent, const xml_node &var_node) -{ - if(!group){ +void DMZ::process_variable(DMR *dmr, D4Group *group, Constructor *parent, const xml_node &var_node) { + if (!group) { throw BESInternalError( - prolog + "Received a null valued Group pointer!", __FILE__, __LINE__); + prolog + "Received a null valued Group pointer!", __FILE__, __LINE__); } string unsupported_flag; - if(d_elide_unsupported && flagged_as_unsupported_type(var_node, unsupported_flag)){ + if (d_elide_unsupported && flagged_as_unsupported_type(var_node, unsupported_flag)) { // And in this way we elide the unsupported types - we don't process the DAP object // if it's got the unsupported bits in fillValue auto var_name = var_node.attribute("name"); auto var_type = var_node.name(); - INFO_LOG(prolog + "Unsupported Type Encountered: " + var_type + " " + var_name.value() + "; flag: '" + unsupported_flag + "'\n"); + INFO_LOG( + prolog + "Unsupported Type Encountered: " + var_type + " " + var_name.value() + "; flag: '" + + unsupported_flag + "'\n"); return; } @@ -512,10 +504,11 @@ void DMZ::process_variable(DMR *dmr, D4Group *group, Constructor *parent, const // Variables are arrays if they have one or more child nodes. Type t = get_type(var_node.name()); - if(t == dods_group_c){ // Groups are special and handled elsewhere + if (t == dods_group_c) { + // Groups are special and handled elsewhere throw BESInternalError( - prolog + "ERROR - The variable node to process is a Group type! " - "This is handled elsewhere, not here. Parser State Issue!!", __FILE__, __LINE__); + prolog + "ERROR - The variable node to process is a Group type! " + "This is handled elsewhere, not here. Parser State Issue!!", __FILE__, __LINE__); } BaseType *btp; @@ -523,16 +516,16 @@ void DMZ::process_variable(DMR *dmr, D4Group *group, Constructor *parent, const // If it has Dim nodes then it's an array! btp = add_array_variable(dmr, group, parent, t, var_node); if (t == dods_structure_c || t == dods_sequence_c) { - if(btp->type() != dods_array_c || btp->var()->type() != t){ + if (btp->type() != dods_array_c || btp->var()->type() != t) { throw BESInternalError( - prolog + "Failed to create an array variable for " + var_node.name(), __FILE__, __LINE__); + prolog + "Failed to create an array variable for " + var_node.name(), __FILE__, __LINE__); } // NB: For an array of a Constructor, add children to the Constructor, not the array - parent = dynamic_cast(btp->var()); - if(!parent){ + parent = dynamic_cast(btp->var()); + if (!parent) { throw BESInternalError( - prolog + "Failed to cast " + btp->var()->type_name() + " " + btp->name() + - " to an instance of Constructor." , __FILE__, __LINE__); + prolog + "Failed to cast " + btp->var()->type_name() + " " + btp->name() + + " to an instance of Constructor.", __FILE__, __LINE__); } for (auto child = var_node.first_child(); child; child = child.next_sibling()) { if (member_of(variable_elements, child.name())) @@ -544,15 +537,15 @@ void DMZ::process_variable(DMR *dmr, D4Group *group, Constructor *parent, const // Things not arrays must be scalars... btp = add_scalar_variable(dmr, group, parent, t, var_node); if (t == dods_structure_c || t == dods_sequence_c) { - if(btp->type() != t){ + if (btp->type() != t) { throw BESInternalError( - prolog + "Failed to create a scalar variable for " + var_node.name(), __FILE__, __LINE__); + prolog + "Failed to create a scalar variable for " + var_node.name(), __FILE__, __LINE__); } - parent = dynamic_cast(btp); - if(!parent){ + parent = dynamic_cast(btp); + if (!parent) { throw BESInternalError( - prolog + "Failed to cast " + btp->var()->type_name() + " " + btp->name() + - " to an instance of Constructor." , __FILE__, __LINE__); + prolog + "Failed to cast " + btp->var()->type_name() + " " + btp->name() + + " to an instance of Constructor.", __FILE__, __LINE__); } for (auto child = var_node.first_child(); child; child = child.next_sibling()) { if (member_of(variable_elements, child.name())) @@ -571,9 +564,8 @@ void DMZ::process_variable(DMR *dmr, D4Group *group, Constructor *parent, const * @param t What DAP type is the new variable? * @param var_node */ -BaseType *DMZ::build_variable(DMR *dmr, D4Group *group, Type t, const xml_node &var_node) -{ - if(!dmr->factory()){ +BaseType *DMZ::build_variable(DMR *dmr, D4Group *group, Type t, const xml_node &var_node) { + if (!dmr->factory()) { throw BESInternalError(prolog + "ERROR - Received a DMR without a class factory!", __FILE__, __LINE__); } @@ -593,7 +585,7 @@ BaseType *DMZ::build_variable(DMR *dmr, D4Group *group, Type t, const xml_node & BaseType *btp = dmr->factory()->NewVariable(t, name_value); if (!btp) - throw BESInternalError("Could not instantiate the variable ' "+ name_value +"'.", __FILE__, __LINE__); + throw BESInternalError("Could not instantiate the variable ' " + name_value + "'.", __FILE__, __LINE__); btp->set_is_dap4(true); @@ -601,16 +593,17 @@ BaseType *DMZ::build_variable(DMR *dmr, D4Group *group, Type t, const xml_node & if (t == dods_enum_c) { if (enum_value.empty()) throw BESInternalError("The variable ' " + name_value + "' lacks an 'enum' attribute.", __FILE__, __LINE__); - D4EnumDef *enum_def =nullptr; + D4EnumDef *enum_def = nullptr; if (enum_value[0] == '/') enum_def = dmr->root()->find_enum_def(enum_value); else enum_def = group->find_enum_def(enum_value); if (!enum_def) - throw BESInternalError("Could not find the Enumeration definition '" + enum_value + "'.", __FILE__, __LINE__); + throw BESInternalError("Could not find the Enumeration definition '" + enum_value + "'.", __FILE__, + __LINE__); - dynamic_cast(*btp).set_enumeration(enum_def); + dynamic_cast(*btp).set_enumeration(enum_def); } return btp; @@ -620,15 +613,14 @@ BaseType *DMZ::build_variable(DMR *dmr, D4Group *group, Type t, const xml_node & * Given that a tag which opens a variable declaration has just been read, * create the variable. This is used when the variable is a scalar. * @param dmr - * @param group If parent is null, add the new var to this group + * @param group If the parent is null, add the new var to this group * @param parent If non-null, add the new var to this constructor * @param t What DAP type is the variable? * @param var_node * @return The new variable */ -BaseType *DMZ::add_scalar_variable(DMR *dmr, D4Group *group, Constructor *parent, Type t, const xml_node &var_node) -{ - if(!group){ +BaseType *DMZ::add_scalar_variable(DMR *dmr, D4Group *group, Constructor *parent, Type t, const xml_node &var_node) { + if (!group) { throw BESInternalError(prolog + "ERROR - Received a null valued Group pointer!", __FILE__, __LINE__); } @@ -658,16 +650,15 @@ BaseType *DMZ::add_scalar_variable(DMR *dmr, D4Group *group, Constructor *parent * @param var_node * @return */ -BaseType *DMZ::add_array_variable(DMR *dmr, D4Group *group, Constructor *parent, Type t, const xml_node &var_node) -{ - if(!group){ +BaseType *DMZ::add_array_variable(DMR *dmr, D4Group *group, Constructor *parent, Type t, const xml_node &var_node) { + if (!group) { throw BESInternalError(prolog + "ERROR - Received a null valued Group pointer!", __FILE__, __LINE__); } BaseType *btp = build_variable(dmr, group, t, var_node); // Transform the scalar to an array - auto *array = static_cast(dmr->factory()->NewVariable(dods_array_c, btp->name())); + auto *array = static_cast(dmr->factory()->NewVariable(dods_array_c, btp->name())); array->set_is_dap4(true); array->add_var_nocopy(btp); @@ -683,7 +674,8 @@ BaseType *DMZ::add_array_variable(DMR *dmr, D4Group *group, Constructor *parent, process_map(dmr, group, array, child); } else if (is_eq(child.name(), DMRPP_FIXED_LENGTH_STRING_ARRAY_ELEMENT)) { - BESDEBUG(PARSER, prolog << "Variable has been marked with a " << DMRPP_FIXED_LENGTH_STRING_ARRAY_ELEMENT << endl); + BESDEBUG(PARSER, + prolog << "Variable has been marked with a " << DMRPP_FIXED_LENGTH_STRING_ARRAY_ELEMENT << endl); // array->set_is_flsa(true); for (xml_attribute attr = child.first_attribute(); attr; attr = attr.next_attribute()) { @@ -694,11 +686,11 @@ BaseType *DMZ::add_array_variable(DMR *dmr, D4Group *group, Constructor *parent, else if (is_eq(attr.name(), DMRPP_FIXED_LENGTH_STRING_PAD_ATTR)) { string_pad_type pad = array->set_fixed_length_string_pad_type(attr.value()); BESDEBUG(PARSER, prolog << "Fixed length string array padding scheme: " << pad << " (" << - array->get_fixed_length_string_pad_str() << ")" << endl); + array->get_fixed_length_string_pad_str() << ")" << endl); } } } - else if(is_eq(child.name(), DMRPP_VLSA_ELEMENT)){ + else if (is_eq(child.name(), DMRPP_VLSA_ELEMENT)) { BESDEBUG(PARSER, prolog << "Variable has been marked with a " << DMRPP_VLSA_ELEMENT << endl); array->set_is_vlsa(true); } @@ -711,14 +703,14 @@ BaseType *DMZ::add_array_variable(DMR *dmr, D4Group *group, Constructor *parent, return array; } + /** * @brief Process an Enumeration element * @param dmr * @param parent * @param var_node */ -void DMZ::process_enum_def(D4Group *d4g, const xml_node &var_node) -{ +void DMZ::process_enum_def(D4Group *d4g, const xml_node &var_node) { string enum_def_name; string basetype_value; for (xml_attribute attr = var_node.first_attribute(); attr; attr = attr.next_attribute()) { @@ -731,26 +723,29 @@ void DMZ::process_enum_def(D4Group *d4g, const xml_node &var_node) } if (enum_def_name.empty()) - throw BESInternalError("The required attribute 'name' was missing from an enumeration element.", __FILE__, __LINE__); + throw BESInternalError("The required attribute 'name' was missing from an enumeration element.", __FILE__, + __LINE__); if (basetype_value.empty()) - throw BESInternalError("The required attribute 'basetype' was missing from an enumeration element.", __FILE__, __LINE__); + throw BESInternalError("The required attribute 'basetype' was missing from an enumeration element.", __FILE__, + __LINE__); Type enum_def_type = get_type(basetype_value.c_str()); if (!is_integer_type(enum_def_type)) { - string err_msg = "The enumeration '" + enum_def_name +"' must have an integer type, instead the type '"+basetype_value; + string err_msg = "The enumeration '" + enum_def_name + "' must have an integer type, instead the type '" + + basetype_value; err_msg += "' is used."; throw BESInternalError(err_msg, __FILE__, __LINE__); } D4EnumDefs *d4enumdefs = d4g->enum_defs(); - vector labels; - vector label_values; + vector labels; + vector label_values; for (auto child = var_node.first_child(); child; child = child.next_sibling()) { - if (is_eq(child.name(),"EnumConst")) { + if (is_eq(child.name(), "EnumConst")) { string enum_const_def_name; string enum_const_def_value; - for (xml_attribute attr =child.first_attribute(); attr; attr = attr.next_attribute()) { + for (xml_attribute attr = child.first_attribute(); attr; attr = attr.next_attribute()) { if (is_eq(attr.name(), "name")) { enum_const_def_name = attr.value(); } @@ -759,21 +754,20 @@ void DMZ::process_enum_def(D4Group *d4g, const xml_node &var_node) } } - if (enum_const_def_name.empty()) + if (enum_const_def_name.empty()) throw BESInternalError("The enum const name is missing.", __FILE__, __LINE__); - if (enum_const_def_value.empty()) + if (enum_const_def_value.empty()) throw BESInternalError("The enum const value is missing.", __FILE__, __LINE__); labels.push_back(enum_const_def_name); label_values.push_back(stoll(enum_const_def_value)); } } - auto enum_def_unique = make_unique(enum_def_name,enum_def_type); + auto enum_def_unique = make_unique(enum_def_name, enum_def_type); auto enum_def = enum_def_unique.get(); - for (unsigned i = 0; i add_value(labels[i],label_values[i]); + for (unsigned i = 0; i < labels.size(); i++) + enum_def->add_value(labels[i], label_values[i]); d4enumdefs->add_enum_nocopy(enum_def_unique.release()); - } @@ -785,8 +779,7 @@ void DMZ::process_enum_def(D4Group *d4g, const xml_node &var_node) * @param parent * @param var_node */ -void DMZ::process_group(DMR *dmr, D4Group *parent, const xml_node &var_node) -{ +void DMZ::process_group(DMR *dmr, D4Group *parent, const xml_node &var_node) { string name_value; for (xml_attribute attr = var_node.first_attribute(); attr; attr = attr.next_attribute()) { if (is_eq(attr.name(), "name")) { @@ -801,7 +794,7 @@ void DMZ::process_group(DMR *dmr, D4Group *parent, const xml_node &var_node) if (!btp) throw BESInternalError("Could not instantiate the Group '" + name_value + "'.", __FILE__, __LINE__); - auto new_group = dynamic_cast(btp); + auto new_group = dynamic_cast(btp); // Need to set this to get the D4Attribute behavior in the type classes // shared between DAP2 and DAP4. jhrg 4/18/13 @@ -837,15 +830,14 @@ void DMZ::process_group(DMR *dmr, D4Group *parent, const xml_node &var_node) * @note Assume the DMZ holds valid DMR++ metadata. * @param dmr Pointer to a DMR instance that should be populated */ -void DMZ::build_thin_dmr(DMR *dmr) -{ +void DMZ::build_thin_dmr(DMR *dmr) { auto xml_root_node = d_xml_doc.first_child(); process_dataset(dmr, xml_root_node); auto root_group = dmr->root(); - auto *dg = dynamic_cast(root_group); + auto *dg = dynamic_cast(root_group); if (!dg) throw BESInternalError("Expected the root group to also be an instance of DmrppD4Group.", __FILE__, __LINE__); @@ -869,27 +861,24 @@ void DMZ::build_thin_dmr(DMR *dmr) // This method will check if any variable in this file can apply the direct IO feature. -// If there is none,a global dio flag will be set to false. By checking the global flag, -// the fileout netCDF module may not need to check every variable in the file to see if -// the direct IO can be applied. +// If there is none,a global dio flag will be set to false. By checking the global flag, +// the fileout netCDF module may not need to check every variable in the file to see if +// the direct IO can be applied. bool DMZ::set_up_all_direct_io_flags_phase_1(DMR *dmr) { - - if (d_xml_doc == nullptr){ + if (d_xml_doc == nullptr) { throw BESInternalError(prolog + "Received a null DMR pointer.", __FILE__, __LINE__); } bool dio_flag_value = set_up_direct_io_flag_phase_1(dmr->root()); - + dmr->set_global_dio_flag(dio_flag_value); return dio_flag_value; - } bool DMZ::set_up_direct_io_flag_phase_1(D4Group *group) { - bool ret_value = false; for (auto i = group->var_begin(), e = group->var_end(); i != e; ++i) { - BESDEBUG("dmrpp","Inside set_up_direct_io_flag: var name is "<<(*i)->name()<name()<type() == dods_array_c) { if (true == set_up_direct_io_flag_phase_1(*i)) { ret_value = true; @@ -897,32 +886,30 @@ bool DMZ::set_up_direct_io_flag_phase_1(D4Group *group) { } } } - + if (ret_value == false) { for (auto gi = group->grp_begin(), ge = group->grp_end(); gi != ge; ++gi) { if (true == set_up_direct_io_flag_phase_1(*gi)) { - ret_value = true; + ret_value = true; break; } } } return ret_value; - } bool DMZ::set_up_direct_io_flag_phase_1(BaseType *btp) { - - // goto the DOM tree node for this variable + // goto the DOM tree node for this variable xml_node var_node = get_variable_xml_node(btp); if (var_node == nullptr) throw BESInternalError("Could not find location of variable in the DMR++ XML document.", __FILE__, __LINE__); auto chunks = var_node.child("dmrpp:chunks"); - if(!chunks) + if (!chunks) return false; bool ret_value = false; - for (xml_attribute attr = chunks.first_attribute(); attr; attr = attr.next_attribute()) { + for (xml_attribute attr = chunks.first_attribute(); attr; attr = attr.next_attribute()) { if (is_eq(attr.name(), "deflateLevel")) { ret_value = true; break; @@ -932,45 +919,41 @@ bool DMZ::set_up_direct_io_flag_phase_1(BaseType *btp) { } void DMZ::set_up_all_direct_io_flags_phase_2(DMR *dmr) { - - if (d_xml_doc == nullptr){ + if (d_xml_doc == nullptr) { throw BESInternalError(prolog + "Received a null DMR pointer.", __FILE__, __LINE__); } set_up_direct_io_flag_phase_2(dmr->root()); - } void DMZ::set_up_direct_io_flag_phase_2(D4Group *group) { - for (auto i = group->var_begin(), e = group->var_end(); i != e; ++i) { - if ((*i)->type() == dods_array_c) - set_up_direct_io_flag_phase_2((*i)); + if ((*i)->type() == dods_array_c) + set_up_direct_io_flag_phase_2((*i)); } - for (auto gi = group->grp_begin(), ge = group->grp_end(); gi != ge; ++gi) - set_up_direct_io_flag_phase_2((*gi)); - + for (auto gi = group->grp_begin(), ge = group->grp_end(); gi != ge; ++gi) + set_up_direct_io_flag_phase_2((*gi)); } void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { - bool is_integer_float = false; Array *t_a = nullptr; - + Type t = btp->type(); if (t == dods_array_c) { - t_a=dynamic_cast(btp); + t_a = dynamic_cast(btp); Type t_var = t_a->var()->type(); - if (libdap::is_simple_type(t_var) && t_var != dods_str_c && t_var != dods_url_c && t_var!= dods_enum_c && t_var!=dods_opaque_c) + if (libdap::is_simple_type(t_var) && t_var != dods_str_c && t_var != dods_url_c && t_var != dods_enum_c && t_var + != dods_opaque_c) is_integer_float = true; } - + // If the var is not an integer or float array, don't support the direct IO. if (is_integer_float == false) return; - + // goto the DOM tree node for this variable xml_node var_node = get_variable_xml_node(btp); if (var_node == nullptr) @@ -978,45 +961,42 @@ void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { auto chunks = var_node.child("dmrpp:chunks"); - // No chunks,no need to check the rest. - if(!chunks) + // No chunks,no need to check the rest. + if (!chunks) return; - - bool has_deflate_filter = false; + + bool has_deflate_filter = false; string filter; - vectordeflate_levels; + vector deflate_levels; bool is_le = false; - for (xml_attribute attr = chunks.first_attribute(); attr; attr = attr.next_attribute()) { + for (xml_attribute attr = chunks.first_attribute(); attr; attr = attr.next_attribute()) { if (!has_deflate_filter && is_eq(attr.name(), "compressionType")) { filter = attr.value(); - if (filter.find("deflate") == string::npos) + if (filter.find("deflate") == string::npos) break; - else + else has_deflate_filter = true; } else if (has_deflate_filter && deflate_levels.empty()) { - if (is_eq(attr.name(), "deflateLevel")) { - string def_lev_str = attr.value(); // decompose the string. - vector def_lev_str_vec = BESUtil::split(def_lev_str, ' ' ); - for (const auto &def_lev:def_lev_str_vec) + vector def_lev_str_vec = BESUtil::split(def_lev_str, ' '); + for (const auto &def_lev: def_lev_str_vec) deflate_levels.push_back(stoul(def_lev)); } - } - else if (is_eq(attr.name(),"byteOrder")) { + else if (is_eq(attr.name(), "byteOrder")) { string endian_str = attr.value(); - if (endian_str=="LE") + if (endian_str == "LE") is_le = true; } - else if (is_eq(attr.name(), "DIO") && is_eq(attr.value(),"off")) { + else if (is_eq(attr.name(), "DIO") && is_eq(attr.value(), "off")) { dc(btp)->set_disable_dio(true); BESDEBUG(PARSER, prolog << "direct IO is disabled : the variable name is: " <name() << endl); } @@ -1026,8 +1006,8 @@ void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { if (!has_deflate_filter || (deflate_levels.empty())) return; - // If the datatype is not little-endian, cannot do the direct IO. return. - // The big-endian IEEE-floating-point data also needs byteswap. So we cannot do direct IO. KY 2024-03-03 + // If the datatype is not little-endian, cannot do the direct IO. return. + // The big-endian IEEE-floating-point data also needs byteswap. So we cannot do direct IO. KY 2024-03-03 if (!is_le) return; @@ -1040,48 +1020,48 @@ void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { #endif // Now we need to read the first child of dmrpp:chunks to obtain the chunk sizes. - vectorchunk_dim_sizes; + vector chunk_dim_sizes; for (auto child = chunks.child("dmrpp:chunkDimensionSizes"); child; child = child.next_sibling()) { if (is_eq(child.name(), "dmrpp:chunkDimensionSizes")) { string chunk_sizes_str = child.child_value(); - vector chunk_sizes_str_vec = BESUtil::split(chunk_sizes_str, ' ' ); - for (const auto &chunk_size:chunk_sizes_str_vec) + vector chunk_sizes_str_vec = BESUtil::split(chunk_sizes_str, ' '); + for (const auto &chunk_size: chunk_sizes_str_vec) chunk_dim_sizes.push_back(stoull(chunk_size)); break; } } - // Since the deflate filter is always associated with the chunk storage, + // Since the deflate filter is always associated with the chunk storage, // the chunkDimensionSizes should always exist for the direct IO case. If not, return. if (chunk_dim_sizes.empty()) return; // Now we need to count the number of children with the name inside the . size_t num_chunks_children = 0; - for (auto child = chunks.first_child(); child; child = child.next_sibling()) + for (auto child = chunks.first_child(); child; child = child.next_sibling()) num_chunks_children++; - // If the only child is dmrpp::chunkDimensionSizes, no chunk is found. This is not direct IO case. - if (num_chunks_children == 1) + // If the only child is dmrpp:chunkDimensionSizes, no chunk is found. This is not direct IO case. + if (num_chunks_children == 1) return; // Now we need to check the special case if the chunk size is greater than the dimension size for any dimension. // If this is the case, we will not use the direct chunk IO since netCDF-4 doesn't allow this. // TODO later, if the dimension is unlimited, this restriction can be lifted. Current dmrpp doesn't store the // unlimited dimension information. - vector dim_sizes; + vector dim_sizes; Array::Dim_iter p = t_a->dim_begin(); while (p != t_a->dim_end()) { - dim_sizes.push_back((unsigned long long)(t_a->dimension_size_ll(p))); + dim_sizes.push_back((unsigned long long) (t_a->dimension_size_ll(p))); p++; } bool chunk_less_dim = true; if (chunk_dim_sizes.size() == dim_sizes.size()) { - for (unsigned int i = 0; i dim_sizes[i]) { - chunk_less_dim = false; - break; + chunk_less_dim = false; + break; } } } @@ -1095,12 +1075,12 @@ void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { // First calculate the number of logical chunks. // Also up to this step, the size of chunk_dim_sizes must be the same as the size of dim_sizes. No need to double check. size_t num_logical_chunks = 1; - for (unsigned int i = 0; iname() << endl); @@ -1113,20 +1093,18 @@ void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { dmrpp_vs_info.filter = filter; // Provide the deflate compression levels. - for (const auto &def_lev:deflate_levels) + for (const auto &def_lev: deflate_levels) dmrpp_vs_info.deflate_levels.push_back(def_lev); - + // Provide the chunk dimension sizes. - for (const auto &chunk_dim:chunk_dim_sizes) + for (const auto &chunk_dim: chunk_dim_sizes) dmrpp_vs_info.chunk_dims.push_back(chunk_dim); t_a->set_var_storage_info(dmrpp_vs_info); t_a->set_dio_flag(); - } - /** * Check to see if the current tag is either an \c Attribute or an \c Alias * start tag. This method is a glorified macro... @@ -1135,8 +1113,7 @@ void DMZ::set_up_direct_io_flag_phase_2(BaseType *btp) { * @param attrs The tag's XML attributes * @return True if the tag was an \c Attribute or \c Alias tag */ -void DMZ::process_attribute(D4Attributes *attributes, const xml_node &dap_attr_node) -{ +void DMZ::process_attribute(D4Attributes *attributes, const xml_node &dap_attr_node) { string name_value; string type_value; for (xml_attribute attr = dap_attr_node.first_attribute(); attr; attr = attr.next_attribute()) { @@ -1149,7 +1126,8 @@ void DMZ::process_attribute(D4Attributes *attributes, const xml_node &dap_attr_n } if (name_value.empty() || type_value.empty()) - throw BESInternalError("The required attribute 'name' or 'type' was missing from an Attribute element.", __FILE__, __LINE__); + throw BESInternalError("The required attribute 'name' or 'type' was missing from an Attribute element.", + __FILE__, __LINE__); if (type_value == "Container") { // Make the new attribute container and add it to current container @@ -1175,18 +1153,18 @@ void DMZ::process_attribute(D4Attributes *attributes, const xml_node &dap_attr_n // Process one or more Value elements for (auto value_elem = dap_attr_node.first_child(); value_elem; value_elem = value_elem.next_sibling()) { if (is_eq(value_elem.name(), "Value")) { - attribute->add_value(value_elem.child_value()); // returns the text of the first data node + attribute->add_value(value_elem.child_value()); // returns the text of the first data node } } - } + } } /** - * @brief Build the lineage of a variable on a stack + * @brief Build the lineage of a variable on a stack. * * This function builds a stack of BaseTypes so that the top-most * variable will be at the top level of the DOM tree (a child of - * the Dataset node) and each successive variable can be found + * the Dataset node). Each successive variable can be found * by looking further in the tree. * * @note Because this function works by starting with the leaf node @@ -1198,8 +1176,7 @@ void DMZ::process_attribute(D4Attributes *attributes, const xml_node &dap_attr_n * @param btp Push thisBaseType on the stack, then push it's ancestors * @param bt The stack */ -void DMZ::build_basetype_chain(BaseType *btp, stack &bt) -{ +void DMZ::build_basetype_chain(BaseType *btp, stack &bt) { auto parent = btp->get_parent(); bt.push(btp); @@ -1208,8 +1185,7 @@ void DMZ::build_basetype_chain(BaseType *btp, stack &bt) build_basetype_chain(parent, bt); } -xml_node DMZ::get_variable_xml_node_helper(const xml_node &/*parent_node*/, stack &/*bt*/) -{ +xml_node DMZ::get_variable_xml_node_helper(const xml_node &/*parent_node*/, stack &/*bt*/) { #if !USE_CACHED_XML_NODE // When we have an array of Structure or Sequence, both the Array and the // Structure BaseType are pushed on the stack. This happens because, for @@ -1223,7 +1199,7 @@ xml_node DMZ::get_variable_xml_node_helper(const xml_node &/*parent_node*/, stac // The DMR XML stores both scalar and array variables using XML elements // named for the cardinal type. For an array, that is the type of the // element, so we use BaseType->var()->type_name() for an Array. - string type_name = bt.top()->type() == dods_array_c ? bt.top()->var()->type_name(): bt.top()->type_name(); + string type_name = bt.top()->type() == dods_array_c ? bt.top()->var()->type_name() : bt.top()->type_name(); string var_name = bt.top()->name(); bt.pop(); @@ -1240,9 +1216,9 @@ xml_node DMZ::get_variable_xml_node_helper(const xml_node &/*parent_node*/, stac } } - return xml_node(); // return an empty node + return xml_node(); // return an empty node #else - return xml_node(); // return an empty node + return xml_node(); // return an empty node #endif } @@ -1252,19 +1228,19 @@ xml_node DMZ::get_variable_xml_node_helper(const xml_node &/*parent_node*/, stac * that corresponds to the DMR++ XML document this class manages). * @return The xml_node pointer */ -xml_node DMZ::get_variable_xml_node(BaseType *btp) -{ +xml_node DMZ::get_variable_xml_node(BaseType *btp) { #if USE_CACHED_XML_NODE auto node = dc(btp)->get_xml_node(); if (node == nullptr) - throw BESInternalError(string("The xml_node for '").append(btp->name()).append("' was not recorded."), __FILE__, __LINE__); + throw BESInternalError(string("The xml_node for '").append(btp->name()).append("' was not recorded."), __FILE__, + __LINE__); return node; #else // load the BaseType objects onto a stack, since we start at the leaf and // go backward using its 'parent' pointer, the order of BaseTypes on the // stack will match the order in the hierarchy of the DOM tree. - stack bt; + stack bt; build_basetype_chain(btp, bt); xml_node dataset = d_xml_doc.first_child(); @@ -1294,8 +1270,7 @@ xml_node DMZ::get_variable_xml_node(BaseType *btp) * @param btp The variable */ void -DMZ::load_attributes(BaseType *btp) -{ +DMZ::load_attributes(BaseType *btp) { if (dc(btp)->get_attributes_loaded()) return; @@ -1322,7 +1297,7 @@ DMZ::load_attributes(BaseType *btp) case dods_structure_c: case dods_sequence_c: case dods_grid_c: { - auto *c = dynamic_cast(btp); + auto *c = dynamic_cast(btp); if (c) { for (auto i = c->var_begin(), e = c->var_end(); i != e; i++) { if ((*i)->type() == dods_array_c) @@ -1345,11 +1320,10 @@ DMZ::load_attributes(BaseType *btp) * @param var_node */ void -DMZ::load_attributes(BaseType *btp, xml_node var_node) const -{ +DMZ::load_attributes(BaseType *btp, xml_node var_node) const { if (dc(btp)->get_attributes_loaded()) return; - + // Attributes for this node will be held in the var_node siblings. // NB: Make an explict call to the BaseType implementation in case // the attributes() method is specialized for this DMR++ code to @@ -1370,15 +1344,14 @@ DMZ::load_attributes(BaseType *btp, xml_node var_node) const * @param constructor */ void -DMZ::load_attributes(Constructor *constructor) -{ - load_attributes(constructor, get_variable_xml_node(constructor)); +DMZ::load_attributes(Constructor *constructor) { + load_attributes(constructor, get_variable_xml_node(constructor)); for (auto i = constructor->var_begin(), e = constructor->var_end(); i != e; ++i) { // Groups are not allowed inside a Constructor - if((*i)->type() == dods_group_c){ + if ((*i)->type() == dods_group_c) { throw BESInternalError( - prolog + "Found a Group as a member of a " + constructor->type_name() + " data type. " + - "This violates the DAP4 data model and cannot be processed!", __FILE__, __LINE__); + prolog + "Found a Group as a member of a " + constructor->type_name() + " data type. " + + "This violates the DAP4 data model and cannot be processed!", __FILE__, __LINE__); } load_attributes(*i); } @@ -1390,7 +1363,8 @@ DMZ::load_attributes(D4Group *group) { if (group->get_parent() == nullptr) { xml_node dataset = d_xml_doc.child("Dataset"); if (!dataset) - throw BESInternalError("Could not find the 'Dataset' element in the DMR++ XML document.", __FILE__, __LINE__); + throw BESInternalError("Could not find the 'Dataset' element in the DMR++ XML document.", __FILE__, + __LINE__); load_attributes(group, dataset); } else { @@ -1402,10 +1376,10 @@ DMZ::load_attributes(D4Group *group) { // Groups are kept under a separate container from variables because they // have a different function than the Structure and Sequence types (Groups // never hold data). - if((*i)->type() == dods_group_c){ + if ((*i)->type() == dods_group_c) { throw BESInternalError( - prolog + "Found a Group instance in the variables collection for Group " + group->name() + ". " + - "This violates the DAP4 data model and cannot be processed!", __FILE__, __LINE__); + prolog + "Found a Group instance in the variables collection for Group " + group->name() + ". " + + "This violates the DAP4 data model and cannot be processed!", __FILE__, __LINE__); } load_attributes(*i); } @@ -1415,9 +1389,8 @@ DMZ::load_attributes(D4Group *group) { } } -void DMZ::load_all_attributes(libdap::DMR *dmr) -{ - if(d_xml_doc == nullptr){ +void DMZ::load_all_attributes(libdap::DMR *dmr) { + if (d_xml_doc == nullptr) { throw BESInternalError(prolog + "Received a null DMR pointer.", __FILE__, __LINE__); } load_attributes(dmr->root()); @@ -1440,41 +1413,41 @@ void DMZ::load_all_attributes(libdap::DMR *dmr) * @param compact The location in the DMR++ of the Base64 encoded values */ void -DMZ::process_compact(BaseType *btp, const xml_node &compact) -{ - +DMZ::process_compact(BaseType *btp, const xml_node &compact) { dc(btp)->set_compact(true); auto char_data = compact.child_value(); if (!char_data) - throw BESInternalError("The dmrpp::compact is missing data values.",__FILE__,__LINE__); + throw BESInternalError("The dmrpp:compact is missing data values.",__FILE__,__LINE__); - std::vector decoded = base64::Base64::decode(char_data); + std::vector decoded = base64::Base64::decode(char_data); // Current not support structure, sequence and grid for compact storage. - if (btp->type()== dods_structure_c || btp->type() == dods_sequence_c || btp->type() == dods_grid_c) - throw BESInternalError("The dmrpp::compact element must be the child of an array or a scalar variable", __FILE__, __LINE__); + if (btp->type() == dods_structure_c || btp->type() == dods_sequence_c || btp->type() == dods_grid_c) + throw BESInternalError("The dmrpp:compact element must be the child of an array or a scalar variable", + __FILE__, __LINE__); // Obtain the datatype for array and scalar. - Type dtype =btp->type(); + Type dtype = btp->type(); bool is_array_subset = false; if (dtype == dods_array_c) { auto *da = dynamic_cast(btp); if (da->is_projected()) is_array_subset = true; - else + else dtype = btp->var()->type(); } if (is_array_subset) { auto *da = dynamic_cast(btp); - process_compact_subset(da,decoded); + process_compact_subset(da, decoded); return; } switch (dtype) { case dods_array_c: - throw BESInternalError("DMR++ document fail: An Array may not be the template for an Array.", __FILE__, __LINE__); + throw BESInternalError("DMR++ document fail: An Array may not be the template for an Array.", __FILE__, + __LINE__); case dods_byte_c: case dods_char_c: @@ -1497,21 +1470,21 @@ DMZ::process_compact(BaseType *btp, const xml_node &compact) case dods_str_c: case dods_url_c: { - std::string str(decoded.begin(), decoded.end()); if (btp->type() == dods_array_c) { auto *array = dynamic_cast(btp); - if(!array){ - throw BESInternalError("Internal state error. Object claims to be array but is not.",__FILE__,__LINE__); + if (!array) { + throw BESInternalError("Internal state error. Object claims to be array but is not.",__FILE__, + __LINE__); } - if(array->is_flsa()){ + if (array->is_flsa()) { // It's an array of Fixed Length Strings auto fls_length = array->get_fixed_string_length(); auto pad_type = array->get_fixed_length_string_pad(); auto str_start = reinterpret_cast(decoded.data()); vector fls_values; - while(fls_values.size() < btp->length_ll()){ - string aValue = DmrppArray::ingest_fixed_length_string(str_start,fls_length, pad_type); + while (fls_values.size() < btp->length_ll()) { + string aValue = DmrppArray::ingest_fixed_length_string(str_start, fls_length, pad_type); fls_values.emplace_back(aValue); str_start += fls_length; } @@ -1523,8 +1496,9 @@ DMZ::process_compact(BaseType *btp, const xml_node &compact) throw BESInternalError("Variable Length Strings are not yet supported.",__FILE__,__LINE__); } } - else {// Scalar - if(btp->type() == dods_str_c) { + else { + // Scalar + if (btp->type() == dods_str_c) { auto *st = static_cast(btp); st->val2buf(&str); st->set_read_p(true); @@ -1534,13 +1508,13 @@ DMZ::process_compact(BaseType *btp, const xml_node &compact) st->val2buf(&str); st->set_read_p(true); } - } break; } default: - throw BESInternalError("Unsupported COMPACT storage variable type in the drmpp handler.", __FILE__, __LINE__); + throw BESInternalError("Unsupported COMPACT storage variable type in the drmpp handler.", __FILE__, + __LINE__); case dods_null_c: break; case dods_structure_c: @@ -1557,45 +1531,44 @@ DMZ::process_compact(BaseType *btp, const xml_node &compact) } void DMZ::process_compact_subset(DmrppArray *da, std::vector &decoded) { - - if (da->var()->type() == dods_str_c || da->var()->type() == dods_url_c) - throw BESInternalError("Currently we don't support the subset for the compacted array of string",__FILE__,__LINE__); + if (da->var()->type() == dods_str_c || da->var()->type() == dods_url_c) + throw BESInternalError("Currently we don't support the subset for the compacted array of string",__FILE__, + __LINE__); int64_t num_buf_bytes = da->width_ll(true); vector buf_bytes; - buf_bytes.resize(num_buf_bytes); + buf_bytes.resize(num_buf_bytes); vector da_dims = da->get_shape(false); unsigned long subset_index = 0; vector subset_pos; - handle_subset(da,da->dim_begin(),subset_index, subset_pos,buf_bytes,decoded); + handle_subset(da, da->dim_begin(), subset_index, subset_pos, buf_bytes, decoded); da->val2buf(reinterpret_cast(buf_bytes.data())); da->set_read_p(true); } -void DMZ::process_vlsa(libdap::BaseType *btp, const pugi::xml_node &vlsa_element) -{ +void DMZ::process_vlsa(libdap::BaseType *btp, const pugi::xml_node &vlsa_element) { //--------------------------------------------------------------------------- // Input Sanitization // We do the QC here and not in all the functions called, like endlessly... // if (btp->type() != dods_array_c) { - throw BESInternalError(prolog + "Received an unexpected "+ btp->type_name() + - " Expected an instance of DmrppArray!", __FILE__, __LINE__); + throw BESInternalError(prolog + "Received an unexpected " + btp->type_name() + + " Expected an instance of DmrppArray!", __FILE__, __LINE__); } auto *array = dynamic_cast(btp); if (!array) { throw BESInternalError("Internal state error. " - "Object claims to be array but is not.", __FILE__, __LINE__); + "Object claims to be array but is not.", __FILE__, __LINE__); } - if(array->var()->type() != dods_str_c && array->var()->type() != dods_url_c){ + if (array->var()->type() != dods_str_c && array->var()->type() != dods_url_c) { throw BESInternalError(prolog + "Internal state error. " "Expected array of dods_str_c, got " + - array->var()->type_name(), __FILE__, __LINE__); + array->var()->type_name(), __FILE__, __LINE__); } - vectorentries; + vector entries; vlsa::read(vlsa_element, entries); array->set_is_vlsa(true); @@ -1604,19 +1577,20 @@ void DMZ::process_vlsa(libdap::BaseType *btp, const pugi::xml_node &vlsa_element } void -DMZ::process_missing_data(BaseType *btp, const xml_node &missing_data) -{ +DMZ::process_missing_data(BaseType *btp, const xml_node &missing_data) { BESDEBUG(PARSER, prolog << "Coming to process_missing_data() " << endl); dc(btp)->set_missing_data(true); auto char_data = missing_data.child_value(); if (!char_data) - throw BESInternalError("The dmrpp::missing_data doesn't contain missing data values.",__FILE__,__LINE__); + throw BESInternalError("The dmrpp:missing_data doesn't contain missing data values.",__FILE__,__LINE__); - std::vector decoded = base64::Base64::decode(char_data); + std::vector decoded = base64::Base64::decode(char_data); - if (btp->type() != dods_array_c && btp->type() !=dods_byte_c) - throw BESInternalError("The dmrpp::missing_data element must be the child of an array or a unsigned char scalar variable", __FILE__, __LINE__); + if (btp->type() != dods_array_c && btp->type() != dods_byte_c) + throw BESInternalError( + "The dmrpp:missing_data element must be the child of an array or a unsigned char scalar variable", + __FILE__, __LINE__); if (btp->type() == dods_byte_c) { auto db = dynamic_cast(btp); @@ -1628,53 +1602,47 @@ DMZ::process_missing_data(BaseType *btp, const xml_node &missing_data) vector result_bytes; - // We need to obtain the total buffer size to retrieve the whole array. + // We need to obtain the total buffer size to retrieve the whole array. // We cannot use width_ll() since it will return the number of selected elements. - auto result_size = (uLongf)(da->get_size(false) *da->prototype()->width()); + auto result_size = (uLongf) (da->get_size(false) * da->prototype()->width()); result_bytes.resize(result_size); - if (da->get_size(false) == 1) - memcpy(result_bytes.data(),decoded.data(),result_size); + if (da->get_size(false) == 1) + memcpy(result_bytes.data(), decoded.data(), result_size); else { int retval = uncompress(result_bytes.data(), &result_size, decoded.data(), decoded.size()); if (retval != 0) - throw BESInternalError("The dmrpp::missing_data - fail to uncompress the mssing data.", __FILE__, __LINE__); + throw BESInternalError("The dmrpp:missing_data - fail to uncompress the mssing data.", __FILE__, __LINE__); } if (da->is_projected()) { - int64_t num_buf_bytes = da->width_ll(true); vector buf_bytes; - buf_bytes.resize(num_buf_bytes); + buf_bytes.resize(num_buf_bytes); vector da_dims = da->get_shape(false); unsigned long subset_index = 0; vector subset_pos; - handle_subset(da,da->dim_begin(),subset_index, subset_pos,buf_bytes,result_bytes); + handle_subset(da, da->dim_begin(), subset_index, subset_pos, buf_bytes, result_bytes); da->val2buf(reinterpret_cast(buf_bytes.data())); - } - else + else da->val2buf(reinterpret_cast(result_bytes.data())); da->set_read_p(true); - } -bool +bool DMZ::supported_special_structure_type_internal(Constructor *var_ctor) { - bool ret_value = true; Constructor::Vars_iter vi = var_ctor->var_begin(); Constructor::Vars_iter ve = var_ctor->var_end(); for (; vi != ve; vi++) { - BaseType *bt = *vi; Type t_bt = bt->type(); // Only support array or scalar of float/int/string. if (libdap::is_simple_type(t_bt) == false) { - if (t_bt != dods_array_c) { ret_value = false; break; @@ -1682,138 +1650,136 @@ DMZ::supported_special_structure_type_internal(Constructor *var_ctor) { else { auto t_a = dynamic_cast(bt); Type t_array_var = t_a->var()->type(); - if (!libdap::is_simple_type(t_array_var) || t_array_var == dods_url_c || t_array_var == dods_enum_c || t_array_var==dods_opaque_c) { + if (!libdap::is_simple_type(t_array_var) || t_array_var == dods_url_c || t_array_var == dods_enum_c || + t_array_var == dods_opaque_c) { ret_value = false; break; } } } - else if (t_bt == dods_url_c || t_bt == dods_enum_c || t_bt==dods_opaque_c) { + else if (t_bt == dods_url_c || t_bt == dods_enum_c || t_bt == dods_opaque_c) { ret_value = false; break; } } return ret_value; - } -bool -DMZ::supported_special_structure_type(BaseType *btp) -{ +bool +DMZ::supported_special_structure_type(BaseType *btp) { bool ret_value = false; Type t = btp->type(); - if ((t == dods_array_c && btp->var()->type() == dods_structure_c) || t==dods_structure_c) { + if ((t == dods_array_c && btp->var()->type() == dods_structure_c) || t == dods_structure_c) { Constructor *var_constructor = nullptr; - if (t==dods_structure_c) - var_constructor = dynamic_cast(btp); - else - var_constructor = dynamic_cast(btp->var()); - if (!var_constructor){ + if (t == dods_structure_c) + var_constructor = dynamic_cast(btp); + else + var_constructor = dynamic_cast(btp->var()); + if (!var_constructor) { throw BESInternalError( - prolog + "Failed to cast " + btp->var()->type_name() + " " + btp->name() + - " to an instance of Constructor." , __FILE__, __LINE__); + prolog + "Failed to cast " + btp->var()->type_name() + " " + btp->name() + + " to an instance of Constructor.", __FILE__, __LINE__); } - - ret_value = supported_special_structure_type_internal(var_constructor); + ret_value = supported_special_structure_type_internal(var_constructor); } return ret_value; - } void -DMZ::process_special_structure_data(BaseType *btp, const xml_node &special_structure_data) -{ +DMZ::process_special_structure_data(BaseType *btp, const xml_node &special_structure_data) { BESDEBUG(PARSER, prolog << "Coming to process_special_structure_data() " << endl); - if (supported_special_structure_type(btp) == false) - throw BESInternalError("The dmrpp::the datatype is not a supported special structure variable", __FILE__, __LINE__); + if (supported_special_structure_type(btp) == false) + throw BESInternalError( + "In dmrpp::process_special_structure_data(): the datatype is not a supported special structure variable", + __FILE__, + __LINE__); auto char_data = special_structure_data.child_value(); if (!char_data) - throw BESInternalError("The dmrpp::special_structure_data doesn't contain special structure data values.",__FILE__,__LINE__); + throw BESInternalError( + "In dmrpp::process_special_structure_data(): the special_structure_data element doesn't contain special structure data values.", + __FILE__,__LINE__); - std::vector values = base64::Base64::decode(char_data); + std::vector values = base64::Base64::decode(char_data); size_t total_value_size = values.size(); - if(btp->type() == dods_array_c) { - + if (btp->type() == dods_array_c) { auto ar = dynamic_cast(btp); - if(ar->is_projected()) - throw BESInternalError("The dmrpp::currently we don't support subsetting of special_structure_data.",__FILE__,__LINE__); + if (ar->is_projected()) + throw BESInternalError( + "In dmrpp::process_special_structure_data(): currently we don't support subsetting of special_structure_data.", + __FILE__,__LINE__); int64_t nelms = ar->length_ll(); size_t values_offset = 0; for (int64_t element = 0; element < nelms; ++element) { - - auto dmrpp_s = dynamic_cast(ar->var()->ptr_duplicate()); - if(!dmrpp_s) + auto dmrpp_s = dynamic_cast(ar->var()->ptr_duplicate()); + if (!dmrpp_s) throw InternalErr(__FILE__, __LINE__, "Cannot obtain the structure pointer."); process_special_structure_data_internal(dmrpp_s, values, total_value_size, values_offset); - ar->set_vec_ll((uint64_t)element,dmrpp_s); + ar->set_vec_ll((uint64_t) element, dmrpp_s); delete dmrpp_s; } } else { - size_t values_offset = 0; - auto dmrpp_s = dynamic_cast(btp); - if(!dmrpp_s) - throw InternalErr(__FILE__, __LINE__, "Cannot obtain the structure pointer."); - process_special_structure_data_internal(dmrpp_s, values , total_value_size, values_offset); + auto dmrpp_s = dynamic_cast(btp); + if (!dmrpp_s) + throw InternalErr(__FILE__, __LINE__, "Cannot obtain the structure pointer."); + process_special_structure_data_internal(dmrpp_s, values, total_value_size, values_offset); } - + btp->set_read_p(true); - } -void DMZ::process_special_structure_data_internal(DmrppStructure * dmrpp_s, std::vector &values , size_t total_value_size, size_t & values_offset){ - - Constructor::Vars_iter vi = dmrpp_s->var_begin(); - Constructor::Vars_iter ve = dmrpp_s->var_end(); - +void DMZ::process_special_structure_data_internal(DmrppStructure *dmrpp_s, std::vector &values, + size_t total_value_size, size_t &values_offset) { + auto vi = dmrpp_s->var_begin(); + auto ve = dmrpp_s->var_end(); for (; vi != ve; vi++) { BaseType *bt = *vi; Type t_bt = bt->type(); - if (libdap::is_simple_type(t_bt) && t_bt != dods_str_c && t_bt != dods_url_c && t_bt!= dods_enum_c && t_bt!=dods_opaque_c) { - + if (is_simple_type(t_bt) && t_bt != dods_str_c && t_bt != dods_url_c && t_bt != dods_enum_c && t_bt != + dods_opaque_c) { BESDEBUG("dmrpp", "var name is: " << bt->name() << "'" << endl); BESDEBUG("dmrpp", "var values_offset is: " << values_offset << "'" << endl); bt->val2buf(values.data() + values_offset); values_offset += bt->width_ll(); } else if (t_bt == dods_str_c) { - BESDEBUG("dmrpp", "var string name is: " << bt->name() << "'" << endl); + BESDEBUG("dmrpp", "var string name is: " << bt->name() << "'" << endl); BESDEBUG("dmrpp", "var string values_offset is: " << values_offset << "'" << endl); if (total_value_size < values_offset) throw InternalErr(__FILE__, __LINE__, "The offset of the retrieved value is out of the boundary."); size_t rest_buf_size = total_value_size - values_offset; - u_int8_t* start_pointer = values.data() + values_offset; - vectortemp_buf; + u_int8_t *start_pointer = values.data() + values_offset; + vector temp_buf; temp_buf.resize(rest_buf_size); - memcpy(temp_buf.data(),(void*)start_pointer,rest_buf_size); + memcpy(temp_buf.data(), (void *) start_pointer, rest_buf_size); // find the index of first ";", the separator - size_t string_stop_index =0; + size_t string_stop_index = 0; vector string_value; - for (size_t i = 0; i decoded_str = base64::Base64::decode(encoded_str); - vector decoded_vec; + string encoded_str(string_value.begin(), string_value.end()); + vector decoded_str = base64::Base64::decode(encoded_str); + vector decoded_vec; decoded_vec.resize(decoded_str.size()); - memcpy(decoded_vec.data(),(void*)decoded_str.data(),decoded_str.size()); - string final_str(decoded_vec.begin(),decoded_vec.end()); + memcpy(decoded_vec.data(), (void *) decoded_str.data(), decoded_str.size()); + string final_str(decoded_vec.begin(), decoded_vec.end()); bt->val2buf(&final_str); - values_offset = values_offset + string_stop_index+1; + values_offset = values_offset + string_stop_index + 1; } else if (t_bt == dods_array_c) { @@ -1822,20 +1788,20 @@ void DMZ::process_special_structure_data_internal(DmrppStructure * dmrpp_s, std auto t_a = dynamic_cast(bt); Type ar_basetype = t_a->var()->type(); - if (libdap::is_simple_type(ar_basetype) && ar_basetype != dods_str_c && ar_basetype != dods_url_c && ar_basetype!= dods_enum_c && ar_basetype!=dods_opaque_c) { + if (libdap::is_simple_type(ar_basetype) && ar_basetype != dods_str_c && ar_basetype != dods_url_c && + ar_basetype != dods_enum_c && ar_basetype != dods_opaque_c) { bt->val2buf(values.data() + values_offset); values_offset += bt->width_ll(); } else if (ar_basetype == dods_str_c) { - - if(total_value_size < values_offset) + if (total_value_size < values_offset) throw InternalErr(__FILE__, __LINE__, "The offset of the retrieved value is out of the boundary."); size_t rest_buf_size = total_value_size - values_offset; - u_int8_t* start_pointer = values.data() + values_offset; - vectortemp_buf; + u_int8_t *start_pointer = values.data() + values_offset; + vector temp_buf; temp_buf.resize(rest_buf_size); - memcpy(temp_buf.data(),(void*)start_pointer,rest_buf_size); + memcpy(temp_buf.data(), (void *) start_pointer, rest_buf_size); int64_t num_ar_elems = t_a->length_ll(); @@ -1846,8 +1812,8 @@ void DMZ::process_special_structure_data_internal(DmrppStructure * dmrpp_s, std unsigned int str_index = 0; size_t string_stop_index = 0; - for (size_t i = 0; i decoded_str = base64::Base64::decode(temp_encoded_str); - vector decoded_vec; + for (size_t i = 0; i < num_ar_elems; i++) { + string temp_encoded_str(encoded_str[i].begin(), encoded_str[i].end()); + vector decoded_str = base64::Base64::decode(temp_encoded_str); + vector decoded_vec; decoded_vec.resize(decoded_str.size()); - memcpy(decoded_vec.data(),(void*)decoded_str.data(),decoded_str.size()); - string temp_final_str(decoded_vec.begin(),decoded_vec.end()); + memcpy(decoded_vec.data(), (void *) decoded_str.data(), decoded_str.size()); + string temp_final_str(decoded_vec.begin(), decoded_vec.end()); final_str[i] = temp_final_str; } - t_a->set_value_ll(final_str,num_ar_elems); - values_offset = values_offset + string_stop_index+1; - + t_a->set_value_ll(final_str, num_ar_elems); + values_offset = values_offset + string_stop_index + 1; } else - throw InternalErr(__FILE__, __LINE__, "The base type of this structure is not integer or float or string. Currently it is not supported."); + throw InternalErr( + __FILE__, __LINE__, + "The base type of this structure is not integer or float or string. Currently it is not supported."); } } dmrpp_s->set_read_p(true); - } - -/** - * @brief Parse a chunk node - * There are several different forms a chunk node can take and this handles - * all of them. - * @param dc - * @param chunk - */ /** * @brief Parse a chunk node * There are several different forms a chunk node can take and this handles @@ -1900,8 +1857,7 @@ void DMZ::process_special_structure_data_internal(DmrppStructure * dmrpp_s, std * @param dc * @param chunk */ -void DMZ::process_chunk(DmrppCommon *dc, const xml_node &chunk) const -{ +void DMZ::process_chunk(DmrppCommon *dc, const xml_node &chunk) const { string href; string trust; string offset; @@ -1938,20 +1894,22 @@ void DMZ::process_chunk(DmrppCommon *dc, const xml_node &chunk) const if (filter_mask.empty()) dc->add_chunk(data_url, dc->get_byte_order(), stoull(size), stoull(offset), chunk_position_in_array); else - dc->add_chunk(data_url, dc->get_byte_order(), stoull(size), stoull(offset), stoul(filter_mask), chunk_position_in_array); + dc->add_chunk(data_url, dc->get_byte_order(), stoull(size), stoull(offset), stoul(filter_mask), + chunk_position_in_array); } else { if (filter_mask.empty()) - dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), chunk_position_in_array); + dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), + chunk_position_in_array); else - dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), stoul(filter_mask), chunk_position_in_array); + dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), stoul(filter_mask), + chunk_position_in_array); } dc->accumlate_storage_size(stoull(size)); } -void DMZ::process_block(DmrppCommon *dc, const xml_node &chunk,unsigned int block_count) const -{ +void DMZ::process_block(DmrppCommon *dc, const xml_node &chunk, unsigned int block_count) const { string href; string trust; string offset; @@ -1973,14 +1931,13 @@ void DMZ::process_block(DmrppCommon *dc, const xml_node &chunk,unsigned int bloc else if (is_eq(attr.name(), "trust") || is_eq(attr.name(), "dmrpp:trust")) { href_trusted = is_eq(attr.value(), "true"); } - } if (offset.empty() || size.empty()) throw BESInternalError("Both size and offset are required for a block node.", __FILE__, __LINE__); if (!href.empty()) { shared_ptr data_url(new http::url(href, href_trusted)); - dc->add_chunk(data_url, dc->get_byte_order(), stoull(size), stoull(offset),true, block_count); + dc->add_chunk(data_url, dc->get_byte_order(), stoull(size), stoull(offset), true, block_count); } else dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), true, block_count); @@ -1995,8 +1952,7 @@ void DMZ::process_block(DmrppCommon *dc, const xml_node &chunk,unsigned int bloc * @param dc * @param chunks */ -void DMZ::process_cds_node(DmrppCommon *dc, const xml_node &chunks) -{ +void DMZ::process_cds_node(DmrppCommon *dc, const xml_node &chunks) { for (auto child = chunks.child("dmrpp:chunkDimensionSizes"); child; child = child.next_sibling()) { if (is_eq(child.name(), "dmrpp:chunkDimensionSizes")) { string sizes = child.child_value(); @@ -2005,25 +1961,23 @@ void DMZ::process_cds_node(DmrppCommon *dc, const xml_node &chunks) } } -static void add_fill_value_information(DmrppCommon *dc, const string &value_string, libdap::Type fv_type) -{ +static void add_fill_value_information(DmrppCommon *dc, const string &value_string, libdap::Type fv_type) { dc->set_fill_value_string(value_string); dc->set_fill_value_type(fv_type); dc->set_uses_fill_value(true); - } - - /** - * A 'dmrpp:chunks' node has a chunkDimensionSizes node and then one or more chunks - * nodes, and they have to be in that order. - * - * @param btp - * @param var_node - * @return True when the dmrpp:chunks element was located, false otherwise. - */ -bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const -{ +} + +/** + * A 'dmrpp:chunks' node has a chunkDimensionSizes node and then one or more chunks + * nodes, and they have to be in that order. + * + * @param btp + * @param var_node + * @return True when the dmrpp:chunks element was located, false otherwise. + */ +bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const { auto chunks = var_node.child("dmrpp:chunks"); - if(!chunks) + if (!chunks) return false; bool has_fill_value = false; @@ -2032,24 +1986,22 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const bool is_multi_lb_chunks = false; for (xml_attribute attr = chunks.first_attribute(); attr; attr = attr.next_attribute()) { - if (is_eq(attr.name(), "compressionType")) { dc(btp)->set_filter(attr.value()); } else if (is_eq(attr.name(), "deflateLevel")) { string def_lev_str = attr.value(); // decompose the string. - vector def_lev_str_vec = BESUtil::split(def_lev_str, ' ' ); + vector def_lev_str_vec = BESUtil::split(def_lev_str, ' '); vector def_levels; - for (const auto &def_lev:def_lev_str_vec) + for (const auto &def_lev: def_lev_str_vec) def_levels.push_back(stoul(def_lev)); dc(btp)->set_deflate_levels(def_levels); } else if (is_eq(attr.name(), "fillValue")) { - // Throws BESInternalError when unsupported types detected. string unsupported_type; - if(flagged_as_unsupported_type(var_node,unsupported_type)){ + if (flagged_as_unsupported_type(var_node, unsupported_type)) { stringstream msg; msg << prolog << "Found a dmrpp:chunk/@fillValue with a value of "; msg << "'" << unsupported_type << "' this means that "; @@ -2060,45 +2012,46 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const has_fill_value = true; // Fill values are only supported for Arrays and scalar numeric datatypes (7/12/22) - if (btp->type()==dods_url_c + if (btp->type() == dods_url_c || btp->type() == dods_sequence_c || btp->type() == dods_grid_c) - throw BESInternalError("Fill Value chunks are unsupported for URL, sequence and grid types.", __FILE__, __LINE__); + throw BESInternalError("Fill Value chunks are unsupported for URL, sequence and grid types.", __FILE__, + __LINE__); if (btp->type() == dods_structure_c) { string fvalue_str = attr.value(); } if (btp->type() == dods_array_c) { - auto array = dynamic_cast(btp); + auto array = dynamic_cast(btp); add_fill_value_information(dc(btp), attr.value(), array->var()->type()); } else add_fill_value_information(dc(btp), attr.value(), btp->type()); } + // TODO I think byteOrder is a required attribute for all but teh DIO case. Not sure. jhrg 12/4/25 else if (is_eq(attr.name(), "byteOrder")) dc(btp)->ingest_byte_order(attr.value()); - - // Here we don't need to check the structOffset attribute if the datatype is not dods_structure_c or array of dods_structure_c. - // But since most variables won't have the structOffset attribute, the code will NOT even go to the following "else if block" after - // looping through the last attribute. So still keep the following implementation. + + // Here we don't need to check the structOffset attribute if the datatype is not dods_structure_c or array of dods_structure_c. + // But since most variables won't have the structOffset attribute, the code will NOT even go to the following "else if block" after + // looping through the last attribute. So still keep the following implementation. else if (is_eq(attr.name(), "structOffset")) { string so_str = attr.value(); // decompose the string. - vector so_str_vec = BESUtil::split(so_str, ' ' ); + vector so_str_vec = BESUtil::split(so_str, ' '); vector struct_offsets; - for (const auto &s_off:so_str_vec) + for (const auto &s_off: so_str_vec) struct_offsets.push_back(stoul(s_off)); dc(btp)->set_struct_offsets(struct_offsets); } // The following only applies to rare cases when handling HDF4, most cases won't even come here. - else if (is_eq(attr.name(),"LBChunk")) { + else if (is_eq(attr.name(), "LBChunk")) { string is_lbchunk_value = attr.value(); if (is_lbchunk_value == "true") { is_multi_lb_chunks = true; dc(btp)->set_multi_linked_blocks_chunk(true); } } - } // reset one_chunk_fillvalue to false if has_fill_value = false @@ -2109,7 +2062,7 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const process_cds_node(dc(btp), chunks); // If child node "dmrpp:chunk" is found, the child node "dmrpp:block" will be not present. - // They are mutual exclusive. + // They are mutual exclusive. bool is_chunked_storage = false; for (auto chunk = chunks.child("dmrpp:chunk"); chunk; chunk = chunk.next_sibling()) { @@ -2119,7 +2072,7 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const } } - if (is_chunked_storage && is_multi_lb_chunks== false) { + if (is_chunked_storage && is_multi_lb_chunks == false) { // Chunks for this node will be held in the var_node siblings. for (auto chunk = chunks.child("dmrpp:chunk"); chunk; chunk = chunk.next_sibling()) { if (is_eq(chunk.name(), "dmrpp:chunk")) { @@ -2128,21 +2081,20 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const } } else { - // Blocks for this node, we need to first check if there is only one block. If this is the case, // we should issue an error. for (auto chunk = chunks.child("dmrpp:block"); chunk; chunk = chunk.next_sibling()) { if (is_eq(chunk.name(), "dmrpp:block")) { block_count++; } - if (block_count >1) + if (block_count > 1) break; } } if (block_count > 0) { if (block_count == 1) throw BESInternalError(" The number of linked block is 1, but it should be > 1.", __FILE__, __LINE__); - if (block_count >1) { + if (block_count > 1) { // set using linked block dc(btp)->set_using_linked_block(); // reset the count to 0 to process the blocks. @@ -2150,7 +2102,8 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const for (auto chunk = chunks.child("dmrpp:block"); chunk; chunk = chunk.next_sibling()) { if (is_eq(chunk.name(), "dmrpp:block")) { process_block(dc(btp), chunk, block_count); - BESDEBUG(PARSER, prolog << "This count of linked block of this variable is: " << block_count << endl); + BESDEBUG(PARSER, + prolog << "This count of linked block of this variable is: " << block_count << endl); block_count++; } } @@ -2158,30 +2111,26 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const } } else if (is_multi_lb_chunks) { - - queue >> mb_index_queue; - vector> offset_length_pair; + queue > > mb_index_queue; + vector > offset_length_pair; // Loop through all the chunks. for (auto chunk = chunks.child("dmrpp:chunk"); chunk; chunk = chunk.next_sibling()) { - // Check the block offset and length for this chunk. - if (is_eq(chunk.name(), "dmrpp:chunk")) - add_mblock_index(chunk, mb_index_queue,offset_length_pair); + if (is_eq(chunk.name(), "dmrpp:chunk")) + add_mblock_index(chunk, mb_index_queue, offset_length_pair); } // This is the last one. mb_index_queue.push(offset_length_pair); // Now we get all the blocks and we will process them. for (auto chunk = chunks.child("dmrpp:chunk"); chunk; chunk = chunk.next_sibling()) { - if (is_eq(chunk.name(), "dmrpp:chunk")) - process_multi_blocks_chunk(dc(btp),chunk, mb_index_queue); + if (is_eq(chunk.name(), "dmrpp:chunk")) + process_multi_blocks_chunk(dc(btp), chunk, mb_index_queue); } dc(btp)->set_multi_linked_blocks_chunk(true); - } return true; - } @@ -2190,10 +2139,9 @@ bool DMZ::process_chunks(BaseType *btp, const xml_node &var_node) const * @param array The Array * @return A vector of integers that holds the size of each dimension of the Array */ -vector DMZ::get_array_dims(Array *array) -{ +vector DMZ::get_array_dims(Array *array) { vector array_dim_sizes; - for (auto i= array->dim_begin(), e = array->dim_end(); i != e; ++i) { + for (auto i = array->dim_begin(), e = array->dim_end(); i != e; ++i) { array_dim_sizes.push_back(array->dimension_size_ll(i)); } @@ -2210,13 +2158,12 @@ vector DMZ::get_array_dims(Array *array) * @param dc A pointer to the DmrppCommon information * @return The number of logical chunks. */ -size_t DMZ::logical_chunks(const vector &array_dim_sizes, const DmrppCommon *dc) -{ - auto const& chunk_dim_sizes = dc->get_chunk_dimension_sizes(); +size_t DMZ::logical_chunks(const vector &array_dim_sizes, const DmrppCommon *dc) { + auto const &chunk_dim_sizes = dc->get_chunk_dimension_sizes(); if (chunk_dim_sizes.size() != array_dim_sizes.size()) { ostringstream oss; oss << "Expected the chunk and array rank to match (chunk: " << chunk_dim_sizes.size() << ", array: " - << array_dim_sizes.size() << ")"; + << array_dim_sizes.size() << ")"; throw BESInternalError(oss.str(), __FILE__, __LINE__); } @@ -2224,7 +2171,7 @@ size_t DMZ::logical_chunks(const vector &array_dim_sizes, c auto i = array_dim_sizes.begin(); for (auto chunk_dim_size: chunk_dim_sizes) { auto array_dim_size = *i++; - num_logical_chunks *= (size_t)ceil((float)array_dim_size / (float)chunk_dim_size); + num_logical_chunks *= (size_t) ceil((float) array_dim_size / (float) chunk_dim_size); } return num_logical_chunks; @@ -2244,9 +2191,8 @@ size_t DMZ::logical_chunks(const vector &array_dim_sizes, c * @return A set where each element is a vector of chunk indices. The number of * elements in the set should equal the number of chunks in the _chunks_ parameter. */ -set< vector > DMZ::get_chunk_map(const vector> &chunks) -{ - set< vector > chunk_map; +set > DMZ::get_chunk_map(const vector > &chunks) { + set > chunk_map; for (auto const &chunk: chunks) { chunk_map.insert(chunk->get_position_in_array()); } @@ -2263,27 +2209,27 @@ set< vector > DMZ::get_chunk_map(const vector &chunk_map, const shape &chunk_shape, - const shape &array_shape, unsigned long long chunk_size, unsigned int struct_size) -{ + const shape &array_shape, unsigned long long chunk_size, unsigned int struct_size) { auto dcp = dc(btp); // Use an Odometer to walk over each potential chunk DmrppChunkOdometer odometer(array_shape, chunk_shape); do { const auto &s = odometer.indices(); if (chunk_map.find(s) == chunk_map.end()) { - // Fill Value chunk // what we need byte order, pia, fill value // We also need to check the user-defined fill value case. - vector> structure_type_element; - bool ret_value = is_simple_dap_structure_scalar_array(btp,structure_type_element); + vector > structure_type_element; + bool ret_value = is_simple_dap_structure_scalar_array(btp, structure_type_element); if (ret_value) { - if (struct_size !=0) - dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), chunk_size, s, struct_size); - else - dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), chunk_size, s, structure_type_element); + if (struct_size != 0) + dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), chunk_size, + s, struct_size); + else + dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), chunk_size, + s, structure_type_element); } - else + else dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), chunk_size, s); } } while (odometer.next()); @@ -2297,8 +2243,7 @@ void DMZ::process_fill_value_chunks(BaseType *btp, const set &chunk_map, * * @param btp The variable */ -void DMZ::load_chunks(BaseType *btp) -{ +void DMZ::load_chunks(BaseType *btp) { if (dc(btp)->get_chunks_loaded()) return; // goto the DOM tree node for this variable @@ -2318,8 +2263,10 @@ void DMZ::load_chunks(BaseType *btp) // Chunked data if (process_chunks(btp, var_node)) { chunks_found = 1; - BESDEBUG(PARSER, prolog << "This variable's chunks storage size is: " << dc(btp)->get_var_chunks_storage_size() << endl); - auto array = dynamic_cast(btp); + BESDEBUG(PARSER, + prolog << "This variable's chunks storage size is: " << dc(btp)->get_var_chunks_storage_size() << endl) + ; + auto array = dynamic_cast(btp); // It's possible to have a chunk, but not have a chunk dimension sizes element // when there is only one chunk (e.g., with HDF5 Contiguous storage). jhrg 5/5/22 if (array && !dc(btp)->get_chunk_dimension_sizes().empty()) { @@ -2334,21 +2281,19 @@ void DMZ::load_chunks(BaseType *btp) unsigned long long chunk_size_bytes = array->var()->width(); // start with the element size in bytes vector s_off = dc(btp)->get_struct_offsets(); if (!s_off.empty()) - chunk_size_bytes = s_off.back(); + chunk_size_bytes = s_off.back(); for (auto dim_size: chunk_shape) chunk_size_bytes *= dim_size; - unsigned int struct_size =(s_off.empty())?0:s_off.back(); + unsigned int struct_size = (s_off.empty()) ? 0 : s_off.back(); process_fill_value_chunks(btp, chunk_map, dc(btp)->get_chunk_dimension_sizes(), - array_shape, chunk_size_bytes,struct_size); + array_shape, chunk_size_bytes, struct_size); // Now we need to check if this var only contains one chunk. - // If yes, we will go ahead to set one_chunk_fill_value be true. + // If yes, we will go ahead to set one_chunk_fill_value be true. // While later in process_chunks(), we will check if fillValue is defined and adjust the value. - if (num_logical_chunks == 1) + if (num_logical_chunks == 1) dc(btp)->set_one_chunk_fill_value(true); dc(btp)->set_processing_fv_chunks(); - - } } // If both chunks and chunk_dimension_sizes are empty, this is contiguous storage @@ -2357,7 +2302,7 @@ void DMZ::load_chunks(BaseType *btp) auto const &array_shape = get_array_dims(array); // Position in array is 0, 0, ..., 0 were the number of zeros is the number of array dimensions - shape pia(0,array_shape.size()); + shape pia(0, array_shape.size()); auto dcp = dc(btp); // Since there is one chunk, the chunk size and array size are one and the same. @@ -2365,74 +2310,74 @@ void DMZ::load_chunks(BaseType *btp) for (auto dim_size: array_shape) array_size_bytes *= dim_size; - if (array->var()->type() == dods_str_c) { - + if (array->var()->type() == dods_str_c) { size_t str_size = dcp->get_fill_value().size(); string fvalue = dcp->get_fill_value(); - + // array size above is in _elements_, multiply by the element width to get bytes // We encounter a special case here. In one NASA file, the fillvalue='\0', so - // when converting to string fillvalue becomes "" and the string size is 0. - // This won't correctly pass the fillvalue buffer downstream. So here we + // when converting to string fillvalue becomes "" and the string size is 0. + // This won't correctly pass the fillvalue buffer downstream. So here we // change the fillvalue to ' ' so that it can sucessfully generate netCDF file via fileout netcdf. - // Also for this special case, the string length is 1. + // Also for this special case, the string length is 1. // KY 2022-12-22 - if(dcp->get_fill_value()=="") { - fvalue =" "; + if (dcp->get_fill_value() == "") { + fvalue = " "; } else - array_size_bytes *=str_size; + array_size_bytes *= str_size; dcp->add_chunk(dcp->get_byte_order(), fvalue, dcp->get_fill_value_type(), array_size_bytes, pia); } else { array_size_bytes *= array->var()->width(); // We also need to check the user-defined fill value case. - vector> structure_type_element; - bool ret_value = is_simple_dap_structure_scalar_array(btp,structure_type_element); - if (ret_value) - dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), array_size_bytes, pia, structure_type_element); - else - dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), array_size_bytes, pia); + vector > structure_type_element; + bool ret_value = is_simple_dap_structure_scalar_array(btp, structure_type_element); + if (ret_value) + dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), + array_size_bytes, pia, structure_type_element); + else + dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), + array_size_bytes, pia); } - } - // This is the case when the scalar variable that holds the fill value with the contiguous storage comes. + // This is the case when the scalar variable that holds the fill value with the contiguous storage comes. // Note we only support numeric datatype now. KY 2022-07-12 - else if (btp->type()!=dods_array_c && dc(btp)->get_immutable_chunks().empty()) { - if (btp->type() == dods_grid_c || btp->type() == dods_sequence_c || btp->type() ==dods_url_c) { + else if (btp->type() != dods_array_c && dc(btp)->get_immutable_chunks().empty()) { + if (btp->type() == dods_grid_c || btp->type() == dods_sequence_c || btp->type() == dods_url_c) { ostringstream oss; - oss << " For scalar variable with the contiguous storage that holds the fillvalue, only numeric" - << " types are supported."; + oss << " For scalar variable with the contiguous storage that holds the fillvalue, only numeric" + << " types are supported."; throw BESInternalError(oss.str(), __FILE__, __LINE__); } shape pia; auto dcp = dc(btp); - if (btp->type() == dods_str_c) { - + if (btp->type() == dods_str_c) { size_t array_size = dcp->get_fill_value().size(); string fvalue = dcp->get_fill_value(); // We encounter a special case here. In one NASA file, the fillvalue='\0', so - // when converting to string fillvalue becomes "" and the string size is 0. - // This won't correctly pass the fillvalue buffer downstream. So here we + // when converting to string fillvalue becomes "" and the string size is 0. + // This won't correctly pass the fillvalue buffer downstream. So here we // change the fillvalue to ' ' so that it can successfully generate netCDF file via fileout netcdf. // KY 2022-12-22 - if(dcp->get_fill_value()=="") { - fvalue =" "; + if (dcp->get_fill_value() == "") { + fvalue = " "; array_size = 1; } dcp->add_chunk(dcp->get_byte_order(), fvalue, dcp->get_fill_value_type(), array_size, pia); } else { - vector> structure_type_element; - bool ret_value = is_simple_dap_structure_scalar_array(btp,structure_type_element); - if (ret_value) - dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), btp->width(), pia, structure_type_element); - else - dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), btp->width(), pia); + vector > structure_type_element; + bool ret_value = is_simple_dap_structure_scalar_array(btp, structure_type_element); + if (ret_value) + dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), + btp->width(), pia, structure_type_element); + else + dcp->add_chunk(dcp->get_byte_order(), dcp->get_fill_value(), dcp->get_fill_value_type(), + btp->width(), pia); } - } } @@ -2460,7 +2405,7 @@ void DMZ::load_chunks(BaseType *btp) special_structure_data_found = 1; process_special_structure_data(btp, special_structure_data); } - + auto vlsa_element = var_node.child(DMRPP_VLSA_ELEMENT); if (vlsa_element) { vlsa_found = 1; @@ -2469,11 +2414,14 @@ void DMZ::load_chunks(BaseType *btp) // Here we (optionally) check that exactly one of the supported types of node was found if (DmrppRequestHandler::d_require_chunks) { - int elements_found = chunks_found + chunk_found + compact_found + vlsa_found + missing_data_found + special_structure_data_found; + int elements_found = chunks_found + chunk_found + compact_found + vlsa_found + missing_data_found + + special_structure_data_found; if (elements_found != 1) { ostringstream oss; - oss << "Expected chunk, chunks or compact or variable length string or missing data or special structure data information in the DMR++ data. Found " << elements_found - << " types of nodes."; + oss << + "Expected chunk, chunks or compact or variable length string or missing data or special structure data information in the DMR++ data. Found " + << elements_found + << " types of nodes."; throw BESInternalError(oss.str(), __FILE__, __LINE__); } } @@ -2481,12 +2429,12 @@ void DMZ::load_chunks(BaseType *btp) dc(btp)->set_chunks_loaded(true); } -bool DMZ::is_simple_dap_structure_scalar_array(BaseType *btp, vector> &structure_type_element) { +/// @} +bool DMZ::is_simple_dap_structure_scalar_array(BaseType *btp, vector > &structure_type_element) { bool ret_value = false; - if (btp->type()==dods_array_c) { - + if (btp->type() == dods_array_c) { auto t_a = dynamic_cast(btp); Type t_array_var = t_a->var()->type(); if (t_array_var == dods_structure_c) { @@ -2494,7 +2442,7 @@ bool DMZ::is_simple_dap_structure_scalar_array(BaseType *btp, vectortype() == dods_structure_c) { + else if (btp->type() == dods_structure_c) { auto t_s = dynamic_cast(btp); ret_value = is_simple_dap_structure_internal(t_s, structure_type_element); } @@ -2502,23 +2450,21 @@ bool DMZ::is_simple_dap_structure_scalar_array(BaseType *btp, vector> &structure_type_element) { - +bool DMZ::is_simple_dap_structure_internal(const Structure *ds, vector > &structure_type_element) { bool ret_value = true; - for (const auto &bt:ds->variables()) { - + for (const auto &bt: ds->variables()) { Type t_bt = bt->type(); - + // Only support array or scalar of float/int. if (t_bt == dods_array_c) { auto t_a = dynamic_cast(bt); Type t_array_var = t_a->var()->type(); if (libdap::is_simple_type(t_array_var) == true && t_array_var != dods_str_c) { - pair temp_pair; - int64_t num_eles= t_a->length_ll(); + pair temp_pair; + int64_t num_eles = t_a->length_ll(); temp_pair.first = t_array_var; - temp_pair.second = (int)(num_eles); + temp_pair.second = (int) (num_eles); structure_type_element.push_back(temp_pair); } else { @@ -2527,7 +2473,7 @@ bool DMZ::is_simple_dap_structure_internal(const Structure *ds, vector temp_pair; + pair temp_pair; temp_pair.first = t_bt; temp_pair.second = 1; structure_type_element.push_back(temp_pair); @@ -2541,10 +2487,10 @@ bool DMZ::is_simple_dap_structure_internal(const Structure *ds, vector & subset_pos, - vector& subset_buf, vector& whole_buf) { - - // Obtain the number of elements in each dimension +void DMZ::handle_subset(DmrppArray *da, libdap::Array::Dim_iter dim_iter, unsigned long &subset_index, + vector &subset_pos, + vector &subset_buf, vector &whole_buf) { + // Obtain the number of elements in each dimension vector da_dims = da->get_shape(false); // Obtain the number of bytes of each element @@ -2560,69 +2506,65 @@ void DMZ::handle_subset(DmrppArray *da, libdap::Array::Dim_iter dim_iter, unsign // The end case for the recursion is dimIter == dim_end(); stride == 1 is an optimization // See the else clause for the general case. if (dim_iter == da->dim_end() && stride == 1) { - // For the start and stop indexes of the subset, get the matching indexes in the whole array. subset_pos.push_back(start); - unsigned long long start_index = INDEX_nD_TO_1D( da_dims,subset_pos); + unsigned long long start_index = INDEX_nD_TO_1D(da_dims, subset_pos); subset_pos.pop_back(); subset_pos.push_back(stop); - unsigned long long stop_index = INDEX_nD_TO_1D( da_dims,subset_pos); + unsigned long long stop_index = INDEX_nD_TO_1D(da_dims, subset_pos); subset_pos.pop_back(); // Copy data block from start_index to stop_index - unsigned char * temp_subset_buf = subset_buf.data() + subset_index*bytes_per_elem; - unsigned char * temp_whole_buf = whole_buf.data() + start_index*bytes_per_elem; - size_t num_bytes_to_copy = (stop_index-start_index+1)*bytes_per_elem; + unsigned char *temp_subset_buf = subset_buf.data() + subset_index * bytes_per_elem; + unsigned char *temp_whole_buf = whole_buf.data() + start_index * bytes_per_elem; + size_t num_bytes_to_copy = (stop_index - start_index + 1) * bytes_per_elem; - memcpy(temp_subset_buf,temp_whole_buf,num_bytes_to_copy); + memcpy(temp_subset_buf, temp_whole_buf, num_bytes_to_copy); // Move the subset_index to the next location. - subset_index = subset_index +(stop_index-start_index+1); - + subset_index = subset_index + (stop_index - start_index + 1); } else { for (uint64_t myDimIndex = start; myDimIndex <= stop; myDimIndex += stride) { - // Is it the last dimension? if (dim_iter != da->dim_end()) { // Nope! Then we recurse to the last dimension to read stuff subset_pos.push_back(myDimIndex); // The recursive function will fill in the subset_pos until the dim_end(). - handle_subset(da,dim_iter,subset_index, subset_pos,subset_buf,whole_buf); + handle_subset(da, dim_iter, subset_index, subset_pos, subset_buf, whole_buf); subset_pos.pop_back(); } else { // We are at the last (innermost) dimension, so it's time to copy values. subset_pos.push_back(myDimIndex); - unsigned int sourceIndex = INDEX_nD_TO_1D( da_dims,subset_pos); + unsigned int sourceIndex = INDEX_nD_TO_1D(da_dims, subset_pos); subset_pos.pop_back(); - unsigned char * temp_subset_buf = subset_buf.data() + subset_index*bytes_per_elem; - unsigned char * temp_whole_buf = whole_buf.data() + sourceIndex*bytes_per_elem; - memcpy(temp_subset_buf,temp_whole_buf,bytes_per_elem); - + unsigned char *temp_subset_buf = subset_buf.data() + subset_index * bytes_per_elem; + unsigned char *temp_whole_buf = whole_buf.data() + sourceIndex * bytes_per_elem; + memcpy(temp_subset_buf, temp_whole_buf, bytes_per_elem); + subset_index++; } } } } -void DMZ::add_mblock_index(const xml_node &chunk, queue>>& mb_index_queue, - vector>& offset_length_pair) const{ - +void DMZ::add_mblock_index(const xml_node &chunk, + queue > > &mb_index_queue, + vector > &offset_length_pair) const { string LBIndex_value; for (xml_attribute attr = chunk.first_attribute(); attr; attr = attr.next_attribute()) { - if (is_eq(attr.name(),"LinkedBlockIndex")) { + if (is_eq(attr.name(), "LinkedBlockIndex")) { LBIndex_value = attr.value(); break; - } + } } // We find the linked blocks in this chunk if (LBIndex_value.empty() == false) { - pair temp_offset_length; // We need to loop through the chunk attributes again to find the offset and length. @@ -2645,24 +2587,24 @@ void DMZ::add_mblock_index(const xml_node &chunk, queue>>& mb_index_queue) const { - +void DMZ::process_multi_blocks_chunk(dmrpp::DmrppCommon *dc, const pugi::xml_node &chunk, + std::queue > > & + mb_index_queue) const { // Follow process_chunk string href; string trust; @@ -2672,33 +2614,33 @@ void DMZ::process_multi_blocks_chunk(dmrpp::DmrppCommon *dc, const pugi::xml_nod string filter_mask; bool href_trusted = false; - // We will only check if the last attribute is the "LinkedBlockIndex". + // We will only check if the last attribute is the "LinkedBlockIndex". // If yes, we will check the "LinkedBlockIndex" value, mark it if it is the first index(0). // If the "LinkedBlockIndex" is not 0, we simply return. The information of this linked block is retrieved from the mb_index_queue already. bool multi_lbs_chunk = false; auto LBI_attr = chunk.last_attribute(); - if (is_eq(LBI_attr.name(),"LinkedBlockIndex")) { + if (is_eq(LBI_attr.name(), "LinkedBlockIndex")) { string LBI_attr_value = LBI_attr.value(); - if (LBI_attr_value =="0") + if (LBI_attr_value == "0") multi_lbs_chunk = true; - else + else return; } - else {// This should happen really rarely, still we try to cover the corner case. We loop through all the attributes and search if Linked BlockIndex is present. + else { + // This should happen really rarely, still we try to cover the corner case. We loop through all the attributes and search if Linked BlockIndex is present. for (xml_attribute attr = chunk.first_attribute(); attr; attr = attr.next_attribute()) { - if (is_eq(LBI_attr.name(),"LinkedBlockIndex")) { + if (is_eq(LBI_attr.name(), "LinkedBlockIndex")) { string LBI_attr_value = LBI_attr.value(); - if (LBI_attr_value =="0") + if (LBI_attr_value == "0") multi_lbs_chunk = true; - else + else return; } } } - + // For linked block cases, as far as we know, we don't need to load fill values as the HDF5 case. So we ignore checking and filling the fillvalue to save performance. for (xml_attribute attr = chunk.first_attribute(); attr; attr = attr.next_attribute()) { - if (is_eq(attr.name(), "offset")) { offset = attr.value(); } @@ -2722,52 +2664,53 @@ void DMZ::process_multi_blocks_chunk(dmrpp::DmrppCommon *dc, const pugi::xml_nod if (offset.empty() || size.empty()) throw BESInternalError("Both size and offset are required for a chunk node.", __FILE__, __LINE__); - if (multi_lbs_chunk) {//The chunk that has linked blocks + if (multi_lbs_chunk) { + //The chunk that has linked blocks - vector> temp_pair; - if (!mb_index_queue.empty()) + vector > temp_pair; + if (!mb_index_queue.empty()) temp_pair = mb_index_queue.front(); if (!href.empty()) { shared_ptr data_url(new http::url(href, href_trusted)); - dc->add_chunk(data_url, dc->get_byte_order(), chunk_position_in_array,temp_pair); + dc->add_chunk(data_url, dc->get_byte_order(), chunk_position_in_array, temp_pair); } else { dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), chunk_position_in_array, temp_pair); } mb_index_queue.pop(); // Remove the processed element - } - else { //General Chunk, not the linked block. + else { + //General Chunk, not the linked block. if (!href.empty()) { shared_ptr data_url(new http::url(href, href_trusted)); dc->add_chunk(data_url, dc->get_byte_order(), stoull(size), stoull(offset), chunk_position_in_array); } else { - dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), chunk_position_in_array); + dc->add_chunk(d_dataset_elem_href, dc->get_byte_order(), stoull(size), stoull(offset), + chunk_position_in_array); } } - - dc->accumlate_storage_size(stoull(size)); + dc->accumlate_storage_size(stoull(size)); } // Return the index of the pos in nD array to the equivalent pos in 1D array -size_t DMZ::INDEX_nD_TO_1D (const std::vector < unsigned long long > &dims, - const std::vector < unsigned long long > &pos) { +size_t DMZ::INDEX_nD_TO_1D(const std::vector &dims, + const std::vector &pos) { // // "int a[10][20][30] // & a[1][2][3] == a + (20*30+1 + 30*2 + 1 *3)" // "int b[10][2] // &b[1][1] == b + (2*1 + 1)" // - if(dims.size () != pos.size ()) - throw InternalErr(__FILE__,__LINE__,"dimension error in INDEX_nD_TO_1D routine."); + if (dims.size() != pos.size()) + throw InternalErr(__FILE__,__LINE__, "dimension error in INDEX_nD_TO_1D routine."); size_t sum = 0; - size_t start = 1; + size_t start = 1; - for (const auto & one_pos:pos) { + for (const auto &one_pos: pos) { size_t m = 1; - for (size_t j = start; j < dims.size (); j++) + for (size_t j = start; j < dims.size(); j++) m *= dims[j]; sum += m * one_pos; start++; @@ -2775,6 +2718,4 @@ size_t DMZ::INDEX_nD_TO_1D (const std::vector < unsigned long long > &dims, return sum; } -/// @} - } // namespace dmrpp