diff --git a/.travis.yml b/.travis.yml index e3283e7a68..27de31830a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -144,22 +144,22 @@ stages: jobs: include: - - stage: build-and-package - name: "check" - script: - - export BES_BUILD=main - # - export RUNTESTFLAGS="-v" - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --enable-developer - - make -j16 && make install && besctl start && make check -j16 && besctl stop + # - stage: build-and-package + # name: "check" + # script: + # - export BES_BUILD=main + # # - export RUNTESTFLAGS="-v" + # - autoreconf --force --install --verbose + # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --enable-developer + # - make -j16 && make install && besctl start && make check -j16 && besctl stop - - stage: build-and-package - name: "distcheck" - script: - - export BES_BUILD=distcheck - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION--enable-developer - - make distcheck -j16 GZIP_ENV=--fast + # - stage: build-and-package + # name: "distcheck" + # script: + # - export BES_BUILD=distcheck + # - autoreconf --force --install --verbose + # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION--enable-developer + # - make distcheck -j16 GZIP_ENV=--fast - stage: build-and-package name: "Enterprise Linux 8 RPMs (via Rocky8)" @@ -178,84 +178,84 @@ jobs: --env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY opendap/rocky8_hyrax_builder:latest /root/travis/travis/build-rpm.sh - - stage: build-and-package - name: "dist" - script: - - export BES_BUILD=srcdist - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --with-build=$BES_BUILD_NUMBER - - make dist -j7 - # Make both a bes-- and bes-snapshot tar.gz. This will simplify - # other operations that use the bes source code. Note that the VERSION file holds a - # string that is the version number that is set by the configure script and the build - # number passed into configure when it is run. jhrg 3/23/21 - - SOURCE_VERSION=$(cat bes_VERSION) - - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz - - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz +# - stage: build-and-package +# name: "dist" +# script: +# - export BES_BUILD=srcdist +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --with-build=$BES_BUILD_NUMBER +# - make dist -j7 +# # Make both a bes-- and bes-snapshot tar.gz. This will simplify +# # other operations that use the bes source code. Note that the VERSION file holds a +# # string that is the version number that is set by the configure script and the build +# # number passed into configure when it is run. jhrg 3/23/21 +# - SOURCE_VERSION=$(cat bes_VERSION) +# - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz +# - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz - - stage: scan - name: "scan bes" - script: - - export BES_BUILD=sonar-bes-framework - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage - - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 - - sonar-scanner -Dproject.settings=sonar-bes-framework.properties -Dsonar.login=$SONAR_LOGIN - - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes | grep "QUALITY GATE PASS" +# - stage: scan +# name: "scan bes" +# script: +# - export BES_BUILD=sonar-bes-framework +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage +# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 +# - sonar-scanner -Dproject.settings=sonar-bes-framework.properties -Dsonar.login=$SONAR_LOGIN +# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes | grep "QUALITY GATE PASS" - - stage: scan - name: "scan bes-modules-1" - script: - - export BES_BUILD=sonar-bes-modules - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage - - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 - - sonar-scanner -Dproject.settings=sonar-bes-modules-1.properties -Dsonar.login=$SONAR_MODULES_LOGIN - - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-modules | grep "QUALITY GATE PASS" +# - stage: scan +# name: "scan bes-modules-1" +# script: +# - export BES_BUILD=sonar-bes-modules +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage +# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 +# - sonar-scanner -Dproject.settings=sonar-bes-modules-1.properties -Dsonar.login=$SONAR_MODULES_LOGIN +# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-modules | grep "QUALITY GATE PASS" - - stage: scan - name: "scan bes-hdf-handlers" - script: - - export BES_BUILD=sonar-bes-hdf-handlers - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage - - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 - - sonar-scanner -Dproject.settings=sonar-bes-hdf-handlers.properties -Dsonar.login=$SONAR_SUBMODULES_LOGIN - # We call the hdf4/5 handlers scan opendap-bes-submodules for historical reasons. jhrg 1/13/22 - - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-submodules | grep "QUALITY GATE PASS" +# - stage: scan +# name: "scan bes-hdf-handlers" +# script: +# - export BES_BUILD=sonar-bes-hdf-handlers +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage +# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 +# - sonar-scanner -Dproject.settings=sonar-bes-hdf-handlers.properties -Dsonar.login=$SONAR_SUBMODULES_LOGIN +# # We call the hdf4/5 handlers scan opendap-bes-submodules for historical reasons. jhrg 1/13/22 +# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-submodules | grep "QUALITY GATE PASS" - - stage: hyrax-olfs-trigger - name: "Hyrax OLFS Trigger" - script: - - export STAGE=hyrax-olfs - - echo $STAGE - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER - - ./travis/trigger-olfs-build.sh +# - stage: hyrax-olfs-trigger +# name: "Hyrax OLFS Trigger" +# script: +# - export STAGE=hyrax-olfs +# - echo $STAGE +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER +# - ./travis/trigger-olfs-build.sh -after_script: - - ./travis/upload-test-results.sh +# after_script: +# - ./travis/upload-test-results.sh -before_deploy: - - export DEPLOY="S3" - # Make sure that we have the target dir... - - mkdir -p $TRAVIS_BUILD_DIR/package; - # Source distribution prep (copies both the 'version' and 'snapshot') - - if test "$BES_BUILD" = "srcdist"; then cp bes-*.tar.gz $TRAVIS_BUILD_DIR/package; fi - # Rocky8 distribution prep - - if test "$BES_BUILD" = "rocky8"; then ./travis/rpm-to-package-dir.sh "el8"; fi - # Check for the stuff... - - ls -l $TRAVIS_BUILD_DIR/package +# before_deploy: +# - export DEPLOY="S3" +# # Make sure that we have the target dir... +# - mkdir -p $TRAVIS_BUILD_DIR/package; +# # Source distribution prep (copies both the 'version' and 'snapshot') +# - if test "$BES_BUILD" = "srcdist"; then cp bes-*.tar.gz $TRAVIS_BUILD_DIR/package; fi +# # Rocky8 distribution prep +# - if test "$BES_BUILD" = "rocky8"; then ./travis/rpm-to-package-dir.sh "el8"; fi +# # Check for the stuff... +# - ls -l $TRAVIS_BUILD_DIR/package -# The deploy section copies the snapshot build product our S3 bucket and to www.opendap.org -deploy: - # Push all build results to our S3 bucket - - provider: s3 - access_key_id: $AWS_ACCESS_KEY_ID - secret_access_key: $AWS_SECRET_ACCESS_KEY - bucket: opendap.travis.build - skip_cleanup: true - local_dir: $TRAVIS_BUILD_DIR/package - on: - all_branches: true - condition: $BES_BUILD =~ ^rocky8|srcdist$ +# # The deploy section copies the snapshot build product our S3 bucket and to www.opendap.org +# deploy: +# # Push all build results to our S3 bucket +# - provider: s3 +# access_key_id: $AWS_ACCESS_KEY_ID +# secret_access_key: $AWS_SECRET_ACCESS_KEY +# bucket: opendap.travis.build +# skip_cleanup: true +# local_dir: $TRAVIS_BUILD_DIR/package +# on: +# all_branches: true +# condition: $BES_BUILD =~ ^rocky8|srcdist$ diff --git a/CMakeLists.txt b/CMakeLists.txt index 20f760feb6..c0664eed3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -262,14 +262,18 @@ add_executable( dap/ShowPathInfoResponseHandler.h dap/TempFile.cc dap/TempFile.h - dap/DapUtils.cc - dap/DapUtils.h + dap/DapUtils.cc + dap/DapUtils.h dapreader/DapModule.cc dapreader/DapModule.h dapreader/DapRequestHandler.cc dapreader/DapRequestHandler.h + aws/SignedUrlCache.cc + aws/SignedUrlCache.h + aws/unit-tests/SignedUrlCacheTest.cc + dispatch/PicoSHA2/example/hasher.cpp dispatch/PicoSHA2/example/interactive_hasher.cpp dispatch/PicoSHA2/test/test.cpp @@ -414,7 +418,7 @@ add_executable( dispatch/BESModuleApp.h dispatch/BESNames.h dispatch/BESNotFoundError.h - dispatch/BESObj.h + dispatch/BESObj.h dispatch/BESPlugin.h dispatch/BESPluginFactory.h dispatch/BESProcIdResponseHandler.cc diff --git a/aws/Makefile.am b/aws/Makefile.am index cab5de4f2a..a715586fba 100644 --- a/aws/Makefile.am +++ b/aws/Makefile.am @@ -5,7 +5,7 @@ AUTOMAKE_OPTIONS = foreign -AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch +AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/http if BES_DEVELOPER AM_CPPFLAGS += -DBES_DEVELOPER @@ -15,6 +15,8 @@ AM_CXXFLAGS= AM_LDFLAGS = include $(top_srcdir)/coverage.mk +LDADD = $(BES_HTTP_LIB) + SUBDIRS = . unit-tests noinst_LTLIBRARIES = libbes_aws.la @@ -47,5 +49,7 @@ cccc: -mkdir $(C4_DIR) cccc --outdir=$(C4_DIR) $(SRCS) $(HDRS) -SRCS = AWS_SDK.cc -HDRS = AWS_SDK.h IAWS_SDK.h +SRCS = AWS_SDK.cc \ + SignedUrlCache.cc +HDRS = AWS_SDK.h IAWS_SDK.h \ + SignedUrlCache.h diff --git a/aws/SignedUrlCache.cc b/aws/SignedUrlCache.cc new file mode 100644 index 0000000000..4c34311790 --- /dev/null +++ b/aws/SignedUrlCache.cc @@ -0,0 +1,524 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES http package, part of the Hyrax data server. + +// Copyright (c) 2025 OPeNDAP, Inc. +// Authors: Nathan Potter , Hannah Robertson +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +// Authors: +// ndp Nathan Potter +// Hannah Robertson + +#include "config.h" + +#include + +#include +#include + +#include "AWS_SDK.h" +#include "TheBESKeys.h" +#include "BESDebug.h" +#include "BESStopWatch.h" +#include "BESUtil.h" +#include "CurlUtils.h" +#include "HttpError.h" +#include "HttpNames.h" +#include "EffectiveUrl.h" +#include "SignedUrlCache.h" + +#include "rapidjson/document.h" + +using namespace std; + +constexpr auto MODULE = "euc"; +constexpr auto MODULE_TIMER = "euc:timer"; +constexpr auto MODULE_DUMPER = "euc:dump"; + +#define prolog std::string("SignedUrlCache::").append(__func__).append("() - ") + +namespace bes { + +/** + * @brief Get the cached signed URL. + * @param url_key Key to a cached signed URL. + * @note This method is not, itself, thread safe. + */ +shared_ptr SignedUrlCache::get_cached_signed_url(string const &url_key) { + shared_ptr signed_url(nullptr); + auto it = d_signed_urls.find(url_key); + if (it != d_signed_urls.end()) { + signed_url = (*it).second; + } + return signed_url; +} + +/** + * @brief Return true if the input occurred before the current time, false otherwise + * @param timestamp_str Timestamp must be formatted as returned by AWS endpoint, YYYY-MM-DD HH:mm:dd+timezone, where timezone is formatted `HH:MM`, e.g. `1980-07-16 18:40:58+00:00` + * @note Return false if `timestamp_str` is not a valid timestamp string. + */ +bool SignedUrlCache::is_timestamp_after_now(std::string const ×tamp_str) { + auto now = std::chrono::system_clock::now(); + auto now_secs = std::chrono::time_point_cast(now); + + auto effective_timestamp_str = timestamp_str; + if (timestamp_str.size() == 25) { + // Hack to handle fact that s3credentials from aws include an + // extra colon in their timezone field + // This changes "1980-07-16 18:40:58+00:00" to "1980-07-16 18:40:58+0000" + effective_timestamp_str = string(timestamp_str).erase(22, 1); + } + + std::tm timestamp_time = {}; + auto time_parse_result = strptime(effective_timestamp_str.c_str(), "%F %T%z", ×tamp_time); + if (time_parse_result == nullptr) { + INFO_LOG(prolog + string("PRE-DEPRECATION WARNING - Retrieved s3 credentials timestamp was not able to be parsed - " + timestamp_str)); + return false; + } + auto timestamp_time_point = std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); + auto timestamp_secs = std::chrono::time_point_cast(timestamp_time_point); + return timestamp_secs > now_secs; +} + +/** + * @brief Get the cached signed URL. + * @param url_key Key to a cached signed URL. + * @note This method is not, itself, thread safe. + */ +shared_ptr SignedUrlCache::retrieve_cached_s3credentials(string const &url_key) { + shared_ptr s3_access_key_tuple(nullptr); + auto it = d_s3credentials_cache.find(url_key); + if (it != d_s3credentials_cache.end()) { + // Is it expired? If so, erase it! + auto timestamp_str = get<3>(*(it->second)); + if (!is_timestamp_after_now(timestamp_str)) { + // Expired! + d_s3credentials_cache.erase(it); + } else { + s3_access_key_tuple = it->second; + } + } + return s3_access_key_tuple; +} + +/** + * Find the terminal (effective) url for the source_url. If the source_url matches the + * skip_regex then it will not be cached. + * + * Unlike EffectiveUrlCache, return nullptr instead of making a new EffectiveUrl(source_url); + * + * @param source_url + * @returns The signed effective URL, nullptr if none able to be created +*/ +shared_ptr SignedUrlCache::get_signed_url(shared_ptr source_url) { + + BESDEBUG(MODULE, prolog << "BEGIN url: " << source_url->str() << endl); + BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); + + // Lock access to the cache, the d_signed_urls map. Released when the lock goes out of scope. + std::lock_guard lock_me(d_cache_lock_mutex); + + if (!is_enabled()) { + BESDEBUG(MODULE, prolog << "CACHE IS DISABLED." << endl); + return nullptr; + } + + // if it's not an HTTP url there is nothing to cache. + if (source_url->str().find(HTTP_PROTOCOL) != 0 && source_url->str().find(HTTPS_PROTOCOL) != 0) { + BESDEBUG(MODULE, prolog << "END Not an HTTP request, SKIPPING." << endl); + return nullptr; + } + + if (!d_skip_regex) { + set_skip_regex(); + } + + if (d_skip_regex) { + size_t match_length = 0; + match_length = d_skip_regex->match(source_url->str().c_str(), (int) source_url->str().size()); + if (match_length == source_url->str().size()) { + BESDEBUG(MODULE, prolog << "END Candidate url matches the " + "no_redirects_regex_pattern [" << d_skip_regex->pattern() << + "][match_length=" << match_length << "] SKIPPING." << endl); + return nullptr; + } + BESDEBUG(MODULE, prolog << "Candidate url: '" << source_url->str() + << "' does NOT match the skip_regex pattern [" << d_skip_regex->pattern() << "]" + << endl); + } else { + BESDEBUG(MODULE, prolog << "The cache_effective_urls_skip_regex() was NOT SET " << endl); + } + + shared_ptr signed_url = get_cached_signed_url(source_url->str()); + bool retrieve_and_cache = !signed_url || signed_url->is_expired(); + + // It not found or expired, (re)load. + if (retrieve_and_cache) { + BESDEBUG(MODULE, prolog << "Acquiring signed URL for " << source_url->str() << endl); + { + BES_STOPWATCH_START(MODULE_TIMER, prolog + "Retrieve and cache signed url for source url: " + source_url->str()); + + // 1. Get requisite s3 data urls... + string s3_url; + string s3credentials_url; + tie(s3_url, s3credentials_url) = retrieve_cached_signed_url_components(source_url->str()); + if (s3_url.empty() || s3credentials_url.empty()) { + return nullptr; + } + + // 2. Get unexpired access credentials from cache or s3credentials endpoint... + auto s3_access_key_tuple = retrieve_cached_s3credentials(s3credentials_url); + if (!s3_access_key_tuple) { + s3_access_key_tuple = get_s3credentials_from_endpoint(s3credentials_url); + } + if (!s3_access_key_tuple) { + return nullptr; + } + + // 3: ...and use them to create a signed url + signed_url = sign_url(s3_url, s3_access_key_tuple); + if (!signed_url) { + return nullptr; + } + d_signed_urls[source_url->str()] = signed_url; + } + BESDEBUG(MODULE, prolog << " source_url: " << source_url->str() << " (" + << (source_url->is_trusted() ? "" : "NOT ") << "trusted)" << endl); + BESDEBUG(MODULE, prolog << "signed_url: " << signed_url->dump() << " (" + << (source_url->is_trusted() ? "" : "NOT ") << "trusted)" << endl); + + + BESDEBUG(MODULE, prolog << "Updated record for " << source_url->str() << " cache size: " + << d_signed_urls.size() << endl); + + // Since we don't want there to be a concurrency issue when we release the lock, we don't + // return the instance of shared_ptr that we placed in the cache. Rather + // we make a clone and return that. It will have its own lifecycle independent of + // the instance we placed in the cache - it can be modified and the one in the cache + // is unchanged. Trusted state was established from source_url when signed_url was + // created in sign_url() + signed_url = make_shared(signed_url); + } else { + // Here we have a !expired instance of a shared_ptr retrieved from the cache. + // Now we need to make a copy to return, inheriting trust from the requesting URL. + signed_url = make_shared(signed_url, source_url->is_trusted()); + } + + BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); + BESDEBUG(MODULE, prolog << "END" << endl); + + return signed_url; +} + +/** + * @brief Store each `s3_url` and `s3credentials_url` for key `key_href_url` + * @note Does not cache anything if any of the three inputs are empty + */ +void SignedUrlCache::cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url) { + if (key_href_url.empty() || s3_url.empty() || s3credentials_url.empty() ) { + // Don't cache either if any is empty. + return; + } + d_href_to_s3_cache[key_href_url] = s3_url; + d_href_to_s3credentials_cache[key_href_url] = s3credentials_url; +} + +/** + * @brief Return pair of (s3_url, s3credentials_url) cached for key_href_url + * @note If key_href_url not in cache, returns pair of empty strings + */ +std::pair SignedUrlCache::retrieve_cached_signed_url_components(const std::string &key_href_url) const { + auto it_s3_url = d_href_to_s3_cache.find(key_href_url); + auto it_s3credentials_url = d_href_to_s3credentials_cache.find(key_href_url); + if (it_s3_url == d_href_to_s3_cache.end() || it_s3credentials_url == d_href_to_s3credentials_cache.end() ) { + INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); + return std::pair("", ""); + } + return std::pair(it_s3_url->second, it_s3credentials_url->second); +} + +/** + * @brief Return credentials tuple for given endpoint url `s3credentials_url`, stores credentials for endpoint in `d_s3credentials_cache` + * @note If credential retrieval fails at any point, returns nullptr and does not cache results + */ +shared_ptr SignedUrlCache::get_s3credentials_from_endpoint(std::string const &s3credentials_url) { + // 1. Get the credentials from TEA + std::string s3credentials_json_string; + try { + BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); + + // Note: this http_get call internally adds edl auth headers, if available + curl::http_get(s3credentials_url, s3credentials_json_string); + } + catch (http::HttpError &http_error) { + string err_msg = prolog + "Encountered an error while " + "attempting to retrieve s3 credentials from TEA. " + http_error.get_message(); + INFO_LOG(err_msg); + return nullptr; + } + if (s3credentials_json_string.empty()) { + string err_msg = prolog + "Unable to retrieve s3 credentials from TEA endpoint " + s3credentials_url; + INFO_LOG(err_msg); + return nullptr; + } + + // 2. Parse the response to pull out the credentials + auto credentials = extract_s3_credentials_from_response_json(s3credentials_json_string); + if (credentials) { + // Store credentials if any were retrieved + d_s3credentials_cache[s3credentials_url] = credentials; + } + return credentials; +} + + +/** + * @brief Extract credentials tuple from json response returned from an s3credentials endpoint + * @note Returns nullptr if input is not valid json or does not contain one of the four requisite + * strings: `accessKeyId`, `secretAccessKey`, `sessionToken`, or `expiration` + * @note Lightly adapted from get_urls_from_granules_umm_json_v1_4 + */ +std::shared_ptr SignedUrlCache::extract_s3_credentials_from_response_json(std::string const &s3credentials_json_string) { + rapidjson::Document s3credentials_response; + s3credentials_response.Parse(s3credentials_json_string.c_str()); + + if (s3credentials_response.HasParseError()) { + return nullptr; + } + + string access_key_id; + string secret_access_key; + string session_token; + string expiration; + + auto itr = s3credentials_response.FindMember("accessKeyId"); + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { + access_key_id = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("secretAccessKey"); + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { + secret_access_key = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("sessionToken"); + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { + session_token = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("expiration"); + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { + expiration = itr->value.GetString(); + } + + if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { + return nullptr; + } + return make_shared(access_key_id, + secret_access_key, + session_token, + expiration); +} + +SignedUrlCache *SignedUrlCache::TheCache() { + // Create a local static object the first time the function is called + static SignedUrlCache instance; + + // Initialize the aws library (must only be once in application!) + bes::AWS_SDK::aws_library_initialize(); + + // TODO: do i need a corresponding + // bes::AWS_SDK::aws_library_shutdown(); + // in the destructor?? + + return &instance; +} + + +/** + * @brief Split `s3_url` into bucket, object strings + */ +std::pair SignedUrlCache::split_s3_url(std::string const &s3_url) { + + // Safety first (even though if we were missing the s3:// prefix, s3_url wouldn't have been extracted from the cmr result in the first place....) + if (s3_url.size() < 6 || s3_url.find("s3://") != 0) { + return std::pair("", ""); + } + + // Get the bucket name by removing prefix "s3://" (which must exist or the path + // wouldn't have been extracted from cmr) and including everything up to the first slash + std::string bucket = s3_url.substr(5, s3_url.find("/")); + + // The object name is everything after the bucket name, not including the first "/" + std::string object = s3_url.substr(s3_url.find(bucket) + 1 + bucket.size()); + + return std::pair(bucket, object); +} + + +/** + * @brief Return difference between expiration time and now, if expiration is in future; 0 otherwise + * @param credentials_expiration_datetime Timestamp must be formatted as returned by AWS endpoint, YYYY-MM-DD HH:mm:dd+timezone, where timezone is formatted `HH:MM`, e.g. `1980-07-16 18:40:58+00:00` + * @note Return 0 if `timestamp_str` is not a valid timestamp string. + * @note Shares code with `is_timestamp_after_now`, could maybe converge + * @param current_time Exposed as parameter to aid in testing; defaults to now() + */ +uint64_t SignedUrlCache::num_seconds_until_expiration(const string &credentials_expiration_datetime, const chrono::system_clock::time_point current_time) { + auto now_secs = std::chrono::time_point_cast(current_time); + + auto effective_timestamp_str = credentials_expiration_datetime; + if (credentials_expiration_datetime.size() == 25) { + // Hack to handle fact that s3credentials from aws include an + // extra colon in their timezone field + // This changes "1980-07-16 18:40:58+00:00" to "1980-07-16 18:40:58+0000" + effective_timestamp_str = string(credentials_expiration_datetime).erase(22, 1); + } + + std::tm timestamp_time = {}; + auto time_parse_result = strptime(effective_timestamp_str.c_str(), "%F %T%z", ×tamp_time); + if (time_parse_result == nullptr) { + INFO_LOG(prolog + string("PRE-DEPRECATION WARNING - Retrieved s3 credentials timestamp was not able to be parsed - " + credentials_expiration_datetime)); + return 0; + } + auto timestamp_time_point = std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); + auto timestamp_secs = std::chrono::time_point_cast(timestamp_time_point); + return timestamp_secs > now_secs ? (timestamp_secs - now_secs).count() : 0; +} + +/** + * @brief Sign `s3_url` with aws credentials in `s3_access_key_tuple`, or nullptr if any part of signing process fails + */ +std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, + std::shared_ptr const s3_access_key_tuple, + std::string aws_region) { + bes::AWS_SDK aws_sdk; + string id = get<0>(*s3_access_key_tuple); + string secret = get<1>(*s3_access_key_tuple); + aws_sdk.initialize_s3_client(aws_region, id, secret); + + string bucket; + string object; + tie(bucket, object) = split_s3_url(s3_url); + if (bucket.empty() || object.empty()) { + return nullptr; + } + + auto expiration_seconds = num_seconds_until_expiration(get<3>(*s3_access_key_tuple)); + if (expiration_seconds == 0) { + // No point in creating a url that is already expired!! + return nullptr; + } + + // Can this call fail? If the aws library isn't initialized, it could through a + // BESInternalFatalError, but the library is initialized in the constructor of SignedUrlCache, + // so if we're here it MUST be initialized. + // As far as I can tell, the internal signing function + // doesn't have a chance to throw.... + const Aws::String url_str = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); + + return make_shared(url_str); +} + +/** + * @brief Return if the cache is enabled, which is set in the bes.conf file + * @note Follows the same settings (and relies on the same bes.conf key) as the EffectiveUrlsCache + */ +bool SignedUrlCache::is_enabled() { + // The first time here, the value of d_enabled is -1. Once we check for it in TheBESKeys + // The value will be 0 (false) or 1 (true) and TheBESKeys will not be checked again. + if (d_enabled < 0) { + string value = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_KEY, "false"); + d_enabled = BESUtil::lowercase(value) == "true"; + } + BESDEBUG(MODULE, prolog << "d_enabled: " << (d_enabled ? "true" : "false") << endl); + return d_enabled; +} + +/** + * @return Set the regex used to skip cache keys, which is set in the bes.conf file + * @note Follows the same settings (and relies on the same bes.conf key) as the EffectiveUrlsCache + */ +void SignedUrlCache::set_skip_regex() { + if (!d_skip_regex) { + string pattern = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_SKIP_REGEX_KEY, ""); + if (!pattern.empty()) { + d_skip_regex.reset(new BESRegex(pattern.c_str())); + } + BESDEBUG(MODULE, prolog << "d_skip_regex: " + << (d_skip_regex ? d_skip_regex->pattern() : "Value has not been set.") << endl); + } +} + +/** + * @brief dumps information about this object + * @param strm C++ i/o stream to dump the information to + */ +void SignedUrlCache::dump(ostream &strm) const { + strm << BESIndent::LMarg << prolog << "(this: " << (void *) this << ")" << endl; + BESIndent::Indent(); + strm << BESIndent::LMarg << "d_skip_regex: " << (d_skip_regex ? d_skip_regex->pattern() : "WAS NOT SET") << endl; + if (!d_signed_urls.empty()) { + strm << BESIndent::LMarg << "signed url list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_signed_urls) { + strm << BESIndent::LMarg << i.first << " --> " << i.second->str() << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "signed url list: EMPTY" << endl; + } + + if (!d_href_to_s3credentials_cache.empty()) { + strm << BESIndent::LMarg << "href-to-s3credentials list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_href_to_s3credentials_cache) { + strm << BESIndent::LMarg << i.first << " --> " << i.second << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "href-to-s3credentials list: EMPTY" << endl; + } + + if (!d_href_to_s3_cache.empty()) { + strm << BESIndent::LMarg << "href-to-s3url list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_href_to_s3_cache) { + strm << BESIndent::LMarg << i.first << " --> " << i.second << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "href-to-s3url list: EMPTY" << endl; + } + + if (!d_s3credentials_cache.empty()) { + strm << BESIndent::LMarg << "s3 credentials list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_s3credentials_cache) { + strm << BESIndent::LMarg << i.first << " --> Expires: " << get<3>(*(i.second)) << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "s3 credentials list: EMPTY" << endl; + } + + BESIndent::UnIndent(); +} + +} // namespace bes diff --git a/aws/SignedUrlCache.h b/aws/SignedUrlCache.h new file mode 100644 index 0000000000..a41ddfa86b --- /dev/null +++ b/aws/SignedUrlCache.h @@ -0,0 +1,127 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES aws package, part of the Hyrax data server. + +// Copyright (c) 2025 OPeNDAP, Inc. +// Authors: Nathan Potter , Hannah Robertson +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +// Authors: +// ndp Nathan Potter +// Hannah Robertson + +#ifndef _bes_aws_SignedUrlCache_h_ +#define _bes_aws_SignedUrlCache_h_ 1 + +#include +#include +#include +#include +#include + +#include "BESObj.h" +#include "BESRegex.h" // for std::unique_ptr + +namespace http { + class EffectiveUrl; + class url; +} + +namespace bes { + +/** + * This is a singleton class. It is used to associate a URL with its "pre-signed" AWS s3 URL. This means that + * a URL is signed locally rather than sent through a potentially large number of external redirect actions, as + * in EffectiveUrlCache.h. This url location plus the requisite AWS signature headers, from which the requested bytes + * are transmitted, is termed the "effective url" and is stored in an in memory cache (std::map) so that later + * requests may skip the external signing service and just get required bytes from the actual source. + */ +class SignedUrlCache : public BESObj { +public: + typedef std::tuple S3AccessKeyTuple; + +private: + SignedUrlCache() = default; + + std::mutex d_cache_lock_mutex; + + std::map> d_signed_urls; + + std::map d_href_to_s3credentials_cache; + std::map d_href_to_s3_cache; + + std::shared_ptr get_s3credentials_from_endpoint(std::string const &s3credentials_url); + static std::shared_ptr extract_s3_credentials_from_response_json(std::string const &s3credentials_json_string); + + std::map> d_s3credentials_cache; + std::shared_ptr retrieve_cached_s3credentials(std::string const &url_key); + static bool is_timestamp_after_now(std::string const ×tamp); + + // URLs that match are not cached. + std::unique_ptr d_skip_regex = nullptr; + + int d_enabled = -1; + + static std::pair split_s3_url(std::string const &s3_url); + static uint64_t num_seconds_until_expiration(const std::string &credentials_expiration_datetime, const std::chrono::system_clock::time_point current_time=std::chrono::system_clock::now()); + std::shared_ptr sign_url(std::string const &s3_url, + std::shared_ptr const s3_access_key_tuple, + std::string aws_region="us-west-2"); + std::shared_ptr get_cached_signed_url(std::string const &url_key); + + void set_skip_regex(); + + bool is_enabled(); + + friend class SignedUrlCacheTest; + +public: + /** @brief Get the singleton SignedUrlCache instance. + * + * This static method returns the instance of this singleton class. + * The implementation will only build one instance of SignedUrlCache and + * thereafter return a pointer to that instance. + * + * Thread safe with C++-11 and greater. + * + * @return A pointer to the SignedUrlCache singleton + */ + static SignedUrlCache *TheCache(); + + SignedUrlCache(const SignedUrlCache &src) = delete; + SignedUrlCache &operator=(const SignedUrlCache &rhs) = delete; + + ~SignedUrlCache() override = default; + + void cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url); + std::pair retrieve_cached_signed_url_components(const std::string &key_href_url) const; + std::shared_ptr get_signed_url(std::shared_ptr source_url); + + void dump(std::ostream &strm) const override; + + std::string dump() const { + std::stringstream sstrm; + dump(sstrm); + return sstrm.str(); + } +}; + +} // namespace bes + +#endif // _bes_aws_SignedUrlCache_h_ + diff --git a/aws/unit-tests/.gitignore b/aws/unit-tests/.gitignore index 3d08aff9e5..a267a42dc9 100644 --- a/aws/unit-tests/.gitignore +++ b/aws/unit-tests/.gitignore @@ -1 +1,2 @@ -AWS_SDK_Test \ No newline at end of file +AWS_SDK_Test +/SignedUrlCacheTest \ No newline at end of file diff --git a/aws/unit-tests/AWS_SDK_Test.cc b/aws/unit-tests/AWS_SDK_Test.cc index af55673486..120df96f0b 100644 --- a/aws/unit-tests/AWS_SDK_Test.cc +++ b/aws/unit-tests/AWS_SDK_Test.cc @@ -357,7 +357,7 @@ class AWS_SDK_Test : public CppUnit::TestFixture { }; CPPUNIT_TEST_SUITE_REGISTRATION(AWS_SDK_Test); -} // namespace http +} // namespace bes int main(int argc, char *argv[]) { return bes_run_tests(argc, argv, "cerr,bes,http") ? 0 : 1; diff --git a/aws/unit-tests/Makefile.am b/aws/unit-tests/Makefile.am index e10c980b76..fa5f94ae68 100644 --- a/aws/unit-tests/Makefile.am +++ b/aws/unit-tests/Makefile.am @@ -2,7 +2,7 @@ AUTOMAKE_OPTIONS = foreign -AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/aws -I$(top_srcdir)/dispatch +AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/aws -I$(top_srcdir)/dispatch -I$(top_srcdir)/http # Help the link step find libs at build time too (not just run time) AM_LDFLAGS = -L$(aws_libdir) @@ -11,7 +11,7 @@ AM_LDFLAGS = -L$(aws_libdir) # like bin_PROGRAMS. # NB: aws_libs is set in configure.ac. jhrg 10/16/25 -LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) $(BES_HTTP_LIB) # aws_libdir is set in configure.ac. jhrg 10/16/25 # aws_libdir64 is also set in configure.ac sbl 11/14/25 @@ -50,19 +50,19 @@ AM_CXXFLAGS = # See above. jhrg 10/10/25 AM_LDFLAGS = include $(top_srcdir)/coverage.mk -DISTCLEANFILES = test_config.h +DISTCLEANFILES = bes.conf test_config.h -CLEANFILES = *.dbg *.log +CLEANFILES = bes.conf *.dbg *.log -EXTRA_DIST = test_config.h.in +EXTRA_DIST = test_config.h.in bes.conf.in check_PROGRAMS = $(UNIT_TESTS) TESTS = $(UNIT_TESTS) -noinst_DATA = +noinst_DATA = bes.conf -BUILT_SOURCES = test_config.h +BUILT_SOURCES = test_config.h bes.conf noinst_HEADERS = test_config.h @@ -77,6 +77,8 @@ test_config.h: $(srcdir)/test_config.h.in Makefile -e "s%[@]abs_top_srcdir[@]%$${mod_abs_top_srcdir}%" \ -e "s%[@]abs_builddir[@]%$${mod_abs_builddir}%" $< > test_config.h +bes.conf: bes.conf.in $(top_srcdir)/configure.ac + %.conf: %.conf.in @clean_abs_top_srcdir=`${PYTHON} -c "import os.path; print(os.path.abspath('${abs_top_srcdir}'))"`; \ sed -e "s%[@]abs_top_srcdir[@]%$$clean_abs_top_srcdir%" \ @@ -88,7 +90,7 @@ test_config.h: $(srcdir)/test_config.h.in Makefile if CPPUNIT -UNIT_TESTS = AWS_SDK_Test +UNIT_TESTS = AWS_SDK_Test SignedUrlCacheTest else @@ -110,3 +112,6 @@ clean-local: AWS_SDK_Test_SOURCES = AWS_SDK_Test.cc AWS_SDK_Test_LDADD = $(LDADD) + +SignedUrlCacheTest_SOURCES = SignedUrlCacheTest.cc +SignedUrlCacheTest_LDADD = $(LDADD) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc new file mode 100644 index 0000000000..9cb141b12e --- /dev/null +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -0,0 +1,400 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES component of the Hyrax Data Server. + +// Copyright (c) 2025 OPeNDAP, Inc. +// Authors: Nathan Potter , Hannah Robertson +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +#include "config.h" + +#include +#include +#include + +#include + +#include "BESError.h" +#include "BESDebug.h" +#include "BESUtil.h" +#include "BESCatalogList.h" +#include "TheBESKeys.h" +#include "BESContextManager.h" + +#include "HttpNames.h" +#include "url_impl.h" +#include "EffectiveUrl.h" +#include "SignedUrlCache.h" + +#include "test_config.h" + +#include "modules/common/run_tests_cppunit.h" + +using namespace std; + +static std::string token; + +#define prolog std::string("SignedUrlCacheTest::").append(__func__).append("() - ") + +namespace bes { + +class SignedUrlCacheTest : public CppUnit::TestFixture { +public: + // Called once before everything gets tested + SignedUrlCacheTest() = default; + + // Called at the end of the tests + ~SignedUrlCacheTest() override = default; + + // Called before each test + void setUp() override { + DBG(cerr << endl); + DBG(cerr << prolog << "BEGIN" << endl); + string bes_conf = BESUtil::assemblePath(TEST_BUILD_DIR, "bes.conf"); + DBG(cerr << prolog << "Using BES configuration: " << bes_conf << endl); + TheBESKeys::ConfigFile = bes_conf; + + // Reset to same starting point every time + // (It's a singleton so resetting it is important for testing determinism) + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + theCache->d_enabled = -1; + + // ...and clear the caches + theCache->d_signed_urls.clear(); + theCache->d_href_to_s3credentials_cache.clear(); + theCache->d_href_to_s3_cache.clear(); + theCache->d_s3credentials_cache.clear(); + + if (!token.empty()) { + DBG(cerr << "Setting BESContext " << EDL_AUTH_TOKEN_KEY << " to: '" << token << "'" << endl); + BESContextManager::TheManager()->set_context(EDL_AUTH_TOKEN_KEY, token); + } + DBG(cerr << prolog << "END" << endl); + } + +/*##################################################################################################*/ +/* TESTS BEGIN */ + + void get_cached_signed_url_test() { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + auto input_url = make_shared("http://started_here.com"); + auto output_url = make_shared("http://started_here.com?signed-now"); + + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(input_url->str(), output_url)); + auto result = SignedUrlCache::TheCache()->get_signed_url(input_url); + CPPUNIT_ASSERT_MESSAGE("Cached url should be retrievable", result->str() == output_url->str()); + + std::string non_http_key("foo"); + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(non_http_key, output_url)); + auto result2 = SignedUrlCache::TheCache()->get_signed_url(make_shared(non_http_key)); + CPPUNIT_ASSERT_MESSAGE("Non-url key returns nullptr", result2 == nullptr); + } + + void is_cache_disabled_test() { + DBG(cerr << prolog << "SignedUrlCache::TheCache()->is_enabled(): " + << (SignedUrlCache::TheCache()->is_enabled() ? "true" : "false") << endl); + CPPUNIT_ASSERT_MESSAGE("Cache is disabled", !SignedUrlCache::TheCache()->is_enabled()); + + auto input_url = make_shared("http://started_here.com"); + auto output_url = make_shared("http://started_here.com?signed-now"); + + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(input_url->str(), output_url)); + CPPUNIT_ASSERT_MESSAGE("Cache contains single item", SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + // When the cache is disabled, we return a nullptr---always. + // (In comparison, the EffectiveUrlCache creates an EffectiveUrl around the raw input url) + auto result_when_disabled = SignedUrlCache::TheCache()->get_signed_url(input_url); + CPPUNIT_ASSERT_MESSAGE("When cache is disabled, nullptr is returned", result_when_disabled == nullptr); + + // ...if we now enable the cache is enabled, we return the previously cached value + SignedUrlCache::TheCache()->d_enabled = true; + CPPUNIT_ASSERT_MESSAGE("Cache is enabled", SignedUrlCache::TheCache()->is_enabled()); + + auto result_when_enabled = SignedUrlCache::TheCache()->get_signed_url(input_url); + CPPUNIT_ASSERT_MESSAGE("When cache is re-enabled, value is returned", result_when_enabled->str() == output_url->str()); + } + + void set_skip_regex_test() { + DBG(cerr << prolog << "BEGIN" << endl); + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + // This one does not add the URL or even check it because it _should_ be matching the skip regex + // in the bes.conf + auto src_url = make_shared("https://foobar.com/opendap/data/nc/fnoc1.nc?dap4.ce=u;v"); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url); + CPPUNIT_ASSERT_MESSAGE("When key matches skip regex, value is not cached", SignedUrlCache::TheCache()->d_signed_urls.empty()); + CPPUNIT_ASSERT_MESSAGE("When key matches skip regex, nullptr is returned", result_url == nullptr); + + // Similarly, skipped even when that url has been previously + // added to the cache somehow + auto output_url = make_shared("http://started_here.com?signed-now"); + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(src_url->str(), output_url)); + auto result_url2 = SignedUrlCache::TheCache()->get_signed_url(src_url); + CPPUNIT_ASSERT_MESSAGE("When key matches skip regex, even if it exists in the cache, the value is not returned", result_url2 == nullptr); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + } + } + + void dump_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + // Add values to each type of subcache + theCache->d_signed_urls.insert( + pair>("www.once.com", make_shared("http://www.upon.com"))); + theCache->d_signed_urls.insert( + pair>("www.a.com", make_shared("http://www.time.com"))); + + auto value = make_shared("a man", "a plan", "a canal", "3035-07-16 02:20:33+00:00"); + theCache->d_s3credentials_cache.insert(pair>("palindrome", value)); + + theCache->d_href_to_s3credentials_cache.insert(pair("foo", "whee")); + theCache->d_href_to_s3_cache.insert(pair("yee", "haw")); + + + // Check to make sure dump includes them + auto strm = std::ostringstream(); + theCache->dump(strm); + // Remove start of string to skip address that varies + auto result = strm.str().substr(49); + std::string expected_str = string("d_skip_regex: ") + + "\n signed url list:" + + // "\n www.foo.com --> http://www.bar.com"; + "\n www.a.com --> http://www.time.com" + + "\n www.once.com --> http://www.upon.com" + + "\n href-to-s3credentials list:" + + "\n foo --> whee" + + "\n href-to-s3url list:" + + "\n yee --> haw" + + "\n s3 credentials list:" + + "\n palindrome --> Expires: 3035-07-16 02:20:33+00:00\n"; + CPPUNIT_ASSERT_MESSAGE("The dump should contain:\n" + expected_str + "\n\nbut did not; INSTEAD was\n" + result, result.find(expected_str) != std::string::npos); + } + + void is_timestamp_after_now_test() { + std::string str_old("1980-07-16 18:40:58+00:00"); + CPPUNIT_ASSERT_MESSAGE("Ancient timestamp is before now", !SignedUrlCache::is_timestamp_after_now(str_old)); + + std::string str_future("2135-07-16 02:20:33+00:00"); + CPPUNIT_ASSERT_MESSAGE("Future timestamp is after now", SignedUrlCache::is_timestamp_after_now(str_future)); + + std::string str_invalid("invalid timestamp woo hooray huzzah"); + CPPUNIT_ASSERT_MESSAGE("Invalid timestamp is not after now", !SignedUrlCache::is_timestamp_after_now(str_invalid)); + } + + void retrieve_cached_s3credentials_test() { + std::string key("i_am_a_key"); + auto result_not_in_cache = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); + CPPUNIT_ASSERT_MESSAGE("Cache miss should return null", result_not_in_cache == nullptr); + + auto value = make_shared("a man", "a plan", "a canal", "2135-07-16 02:20:33+00:00"); + SignedUrlCache::TheCache()->d_s3credentials_cache.insert(pair>(key, value)); + auto result_in_cache = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); + CPPUNIT_ASSERT_MESSAGE("Cache hit should successfully retrieve result", result_in_cache == value); + } + + void retrieve_cached_s3credentials_expired_credentials_test() { + std::string key("i_am_a_key"); + std::string expiration_time("1980-07-16 18:40:58+00:00"); + auto value = make_shared("https://www.foo", "https://www.bar", "https://www.bat", expiration_time); + SignedUrlCache::TheCache()->d_s3credentials_cache.insert(pair>(key, value)); + + auto result = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); + CPPUNIT_ASSERT_MESSAGE("Cached expired result should not be retrieved", result == nullptr); + CPPUNIT_ASSERT_MESSAGE("Expired result should have been removed from cache", SignedUrlCache::TheCache()->d_s3credentials_cache.empty()); + } + + void extract_s3_credentials_from_response_json_test() { + + std::string access_key("i_am_an_access_key_id"); + std::string valid_response( + string("{\n\"accessKeyId\": \"") + access_key + "\",\n" + + "\"secretAccessKey\": \"i_am_a_secret_access_key\",\n" + + "\"sessionToken\": \"i_am_a_fake_token\",\n" + + "\"expiration\": \"3025-09-30 18:40:58+00:00\"\n" + + "} "); + auto result = SignedUrlCache::extract_s3_credentials_from_response_json(valid_response); + CPPUNIT_ASSERT_MESSAGE("Valid json should not return nullptr", result != nullptr); + CPPUNIT_ASSERT_MESSAGE("Access key should be returned as first value", access_key == get<0>(*result)); + + CPPUNIT_ASSERT_MESSAGE("Empty string should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json("") == nullptr); + CPPUNIT_ASSERT_MESSAGE("Invalid json should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json("{foo}") == nullptr); + + std::string invalid_response( + string("{\n\"accessKeyId\": \"") + access_key + "\",\n" + + "\"secretAccessKey\": \"i_am_a_secret_access_key\",\n" + + "\"sessionToken\": \"i_am_a_fake_token\",\n" + + "} "); + CPPUNIT_ASSERT_MESSAGE("Response missing field should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json(invalid_response) == nullptr); + + std::string invalid_response_contents( + string("{\n\"accessKeyId\": \"") + access_key + "\",\n" + + "\"secretAccessKey\": [3, 4, 5],\n" + // Oh no! An array instead of a string!! Horrors! + "\"sessionToken\": \"i_am_a_fake_token\",\n" + + "\"expiration\": \"3025-09-30 18:40:58+00:00\"\n" + + "} "); + CPPUNIT_ASSERT_MESSAGE("Field with non-string response should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json(invalid_response_contents) == nullptr); + } + + void cache_signed_url_components_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + theCache->cache_signed_url_components("", "foo", "bar"); + CPPUNIT_ASSERT_MESSAGE("Empty key_href_url results in no caching", theCache->d_href_to_s3_cache.empty() && theCache->d_href_to_s3credentials_cache.empty()); + + theCache->cache_signed_url_components("foo", "", "bar"); + CPPUNIT_ASSERT_MESSAGE("Empty s3_url results in no caching", theCache->d_href_to_s3_cache.empty() && theCache->d_href_to_s3credentials_cache.empty()); + + theCache->cache_signed_url_components("foo", "bar", ""); + CPPUNIT_ASSERT_MESSAGE("Empty s3credentials_url results in no caching", theCache->d_href_to_s3_cache.empty() && theCache->d_href_to_s3credentials_cache.empty()); + + theCache->cache_signed_url_components("foo", "bar", "bat"); + CPPUNIT_ASSERT_MESSAGE("s3_url should be cached", theCache->d_href_to_s3_cache["foo"] == "bar"); + CPPUNIT_ASSERT_MESSAGE("s3credentials_url should be cached", theCache->d_href_to_s3credentials_cache["foo"] == "bat"); + } + + void retrieve_cached_signed_url_components_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + theCache->cache_signed_url_components("two fish", "red fish", "blue fish"); + auto retrieved = theCache->retrieve_cached_signed_url_components("two fish"); + auto expected = std::pair("red fish", "blue fish"); + CPPUNIT_ASSERT_MESSAGE("Cached components should be retrieved", expected == retrieved); + + std::string key("little_bo_peep"); + theCache->d_href_to_s3_cache.insert(pair(key, "goat")); + auto retrieved2 = theCache->retrieve_cached_signed_url_components(key); + auto empty_pair = std::pair("", ""); + CPPUNIT_ASSERT_MESSAGE("If both urls were not cached, no response is returned", retrieved2 == empty_pair); + } + + void get_s3credentials_from_endpoint_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + CPPUNIT_ASSERT_MESSAGE("Credentials cache is empty", theCache->d_s3credentials_cache.empty()); + + auto result_bad = theCache->get_s3credentials_from_endpoint("http://badurl"); + CPPUNIT_ASSERT_MESSAGE("After attempting fetch from invalid url, credentials cache is still empty", theCache->d_s3credentials_cache.empty()); + CPPUNIT_ASSERT_MESSAGE("Fetch from invalid url returns nullptr", result_bad == nullptr); + + // Note: we do not test a successful endpoint here, as that would require either + // relying on TEA or setting up a test url with fake credentials, + // both of which are brittle in their own ways. + // The inputs/outputs for good and bad retrieval are covered by other tests + } + + void split_s3_url_test() { + auto out1 = SignedUrlCache::split_s3_url("s3://foo/bar"); + auto expected1 = std::pair("foo", "bar"); + CPPUNIT_ASSERT_MESSAGE("Valid input is split into `" + get<0>(expected1) + "," + get<1>(expected1) + "`, got `" + get<0>(out1) + "," + get<1>(out1) + "`", expected1 == out1); + + auto expected_empty = std::pair("", ""); + auto out2 = SignedUrlCache::split_s3_url("foo/bar"); + CPPUNIT_ASSERT_MESSAGE("Missing s3:// prefix should return empty strings; returned `" + get<0>(out2) + "," + get<1>(out2) + "`", expected_empty == out2); + auto out3 = SignedUrlCache::split_s3_url("x"); + CPPUNIT_ASSERT_MESSAGE("Missing s3:// prefix should return empty strings; returned `" + get<0>(out3) + "," + get<1>(out3) + "`", expected_empty == out2); + + auto out4 = SignedUrlCache::split_s3_url("s3://foo/bar/wheeyay.txt"); + auto expected4 = std::pair("foo", "bar/wheeyay.txt"); + CPPUNIT_ASSERT_MESSAGE("Valid input is split into `" + get<0>(expected4) + "," + get<1>(expected4) + "`, got `" + get<0>(out4) + "," + get<1>(out4) + "`", expected4 == out4); + } + + std::chrono::system_clock::time_point parse_as_time_point(const std::string& datetime_string) { + std::tm timestamp_time = {}; + strptime(datetime_string.c_str(), "%F %T%z", ×tamp_time); + return std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); + } + + void num_seconds_until_expiration_test() { + std::string current_time_str = "2025-04-01 09:30:00+0400"; + std::string current_time_aws_str = "2025-04-01 09:30:00+04:00"; + std::chrono::system_clock::time_point current_time = parse_as_time_point(current_time_str); + + auto out = SignedUrlCache::num_seconds_until_expiration(current_time_aws_str, current_time); + CPPUNIT_ASSERT_MESSAGE("Expiration time `now` should return 0s` for `" + current_time_aws_str + "`: `" + to_string(out) + "`", out == 0); + + auto time_plus_five_seconds_aws_str = "2025-04-01 09:30:05+04:00"; + auto out1 = SignedUrlCache::num_seconds_until_expiration(time_plus_five_seconds_aws_str, current_time); + CPPUNIT_ASSERT_MESSAGE("Expiration date should be 5s from now `" + to_string(out1) + "`", out1 == 5); + + auto time_minus_five_seconds_aws_str = "2025-04-01 09:29:55+04:00"; + auto out2 = SignedUrlCache::num_seconds_until_expiration(time_minus_five_seconds_aws_str, current_time); + CPPUNIT_ASSERT_MESSAGE("Expiration date in the past should return 0s `" + to_string(out2) + "`", out2 == 0); + + std::string expiration_time_past("1980-07-16 18:40:58+04:00"); + auto out3 = SignedUrlCache::num_seconds_until_expiration(expiration_time_past); + CPPUNIT_ASSERT_MESSAGE("Expiration time in past should return 0s `" + to_string(out3) + "`", out3 == 0); + + std::string expiration_time_future("2126-07-16 18:40:58+04:00"); + auto out4 = SignedUrlCache::num_seconds_until_expiration(expiration_time_future); + CPPUNIT_ASSERT_MESSAGE("Expiration time in future should be > 0s `" + to_string(out4) + "`", out4 > 0); + + auto out5 = SignedUrlCache::num_seconds_until_expiration("lil_date"); + CPPUNIT_ASSERT_MESSAGE("Invalid date string should return 0s `" + to_string(out5) + "`", out5 == 0); + } + +/* TESTS END */ +/*##################################################################################################*/ + +CPPUNIT_TEST_SUITE(SignedUrlCacheTest); + + // Test behavior analogous to that of the EffectiveUrlCache: + CPPUNIT_TEST(get_cached_signed_url_test); + CPPUNIT_TEST(is_cache_disabled_test); + CPPUNIT_TEST(set_skip_regex_test); + CPPUNIT_TEST(dump_test); + + // Test behavior specific to SignedUrlCache: + CPPUNIT_TEST(is_timestamp_after_now_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); + CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + CPPUNIT_TEST(cache_signed_url_components_test); + CPPUNIT_TEST(retrieve_cached_signed_url_components_test); + CPPUNIT_TEST(get_s3credentials_from_endpoint_test); + + // // ...and, specifically, the signing itself: + // // TODO-future: will add/update these tests once signing behavior is implemented + // // - sign_url + // // - get_signed_url + + // Last but not least, test those helper functions + CPPUNIT_TEST(split_s3_url_test); + CPPUNIT_TEST(num_seconds_until_expiration_test); + + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SignedUrlCacheTest); + +} // namespace bes + +int main(int argc, char *argv[]) { + return bes_run_tests(argc, argv, "cerr,bes,http") ? 0 : 1; +} \ No newline at end of file diff --git a/aws/unit-tests/bes.conf.in b/aws/unit-tests/bes.conf.in new file mode 100644 index 0000000000..fc749610fc --- /dev/null +++ b/aws/unit-tests/bes.conf.in @@ -0,0 +1,21 @@ + + +BES.Catalog.catalog.RootDirectory=@abs_top_srcdir@/aws +BES.Catalog.catalog.TypeMatch+=nc:.*\.nc(4)?(\.bz2|\.gz|\.Z)?$ + +BES.LogName = ./bes.log + +############################################################################### +# +# EffectiveUrlCache(Test) configuration. + +# Since many URLs result in a number of time consuming redirects (ex: OAuth2) +# We cache the "effective URL" for each to improve speed and reduce +# authentication churn +Http.cache.effective.urls = false + +# But we also know that many URLs (ex: AWS S3) will never redirect so we can +# skip the caching for those destinations. Any URL matching these patterns will +# not have ensuing redirects followed and cached. +Http.cache.effective.urls.skip.regex.pattern = ^https://foobar\.com/.*$ + diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd new file mode 100644 index 0000000000..aea7d0b5a6 --- /dev/null +++ b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd @@ -0,0 +1,14 @@ + + + 300 + no + xml + http://localhost:8080/opendap/ngap/collections/C2532426483-ORNL_CLOUD/granules/Daymet_Daily_V4R1.daymet_v4_daily_na_tmax_2010.nc + 0 + 0 + collections/C2532426483-ORNL_CLOUD/granules/Daymet_Daily_V4R1.daymet_v4_daily_na_tmax_2010.nc + + + + + diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline new file mode 100644 index 0000000000..949e1e8888 --- /dev/null +++ b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline @@ -0,0 +1,27713 @@ + + + + + + + + + + m + + + y coordinate of projection + + + projection_y_coordinate + + + 8075 + + + + + + + + degrees_east + + + longitude coordinate + + + longitude + + + 1010 977 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + degrees_north + + + latitude coordinate + + + latitude + + + 1010 977 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + time + + + standard + + + days since 1950-01-01 00:00:00 + + + time_bnds + + + 24-hour day based on local time + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + m + + + x coordinate of projection + + + projection_x_coordinate + + + 7814 + + + + + + + + + -9999. + + + daily maximum temperature + + + degrees C + + + -9999. + + + lat lon + + + lambert_conformal_conic + + + area: mean time: maximum + + + + + + 1 1000 1000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + lambert_conformal_conic + + + -100. + + + 42.5 + + + 0. + + + 0. + + + 25. + 60. + + + 6378137. + + + 298.25722356300003 + + + + + + + + + + + 1 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + day of year (DOY) starting with day 1 on Januaray 1st + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2010 + + + Daymet Software Version 4.0 + + + Daymet Software Version 4.0 + + + Daymet Data Version 4.0 + + + CF-1.6 + + + Please see http://daymet.ornl.gov/ for current Daymet data citation information + + + Please see http://daymet.ornl.gov/ for current information on Daymet references + + + + 2025-09-15T19:37:22Z + + + 3.21.1-367 + + + 3.21.1-367 + + + libdap-3.21.1-99 + + + build_dmrpp -f /tmp/tmp0tv4iczy/daymet_v4_daily_na_tmax_2010.nc -r daymet_v4_daily_na_tmax_2010.nc.dmr -u OPeNDAP_DMRpp_DATA_ACCESS_URL -M + + + diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline_alt b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline_alt new file mode 100644 index 0000000000..7bc1e7d756 --- /dev/null +++ b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline_alt @@ -0,0 +1,20 @@ + + + + + 7 + + support@opendap.org + 0 + 401 + https://data.ornldaac.earthdata.nasa.gov/protected/daymet/Daymet_Daily_V4R1/data/daymet_v4_daily_na_tmax_2010.nc.dmrpp + + + + + CurlUtils.cc + 856 + + + + \ No newline at end of file diff --git a/cmdln/tests/testsuite.at b/cmdln/tests/testsuite.at index 8763c78145..2864fe4efd 100644 --- a/cmdln/tests/testsuite.at +++ b/cmdln/tests/testsuite.at @@ -101,6 +101,36 @@ _AT_BESCMD_TEST($abs_srcdir/$1, $abs_srcdir/$1.baseline, $2) AT_CLEANUP ]) +dnl Either match the expected dmrpp result (if EDL credentials present) +dnl or match with the error type returned, without any urls +dnl that could leak credentials or be unmatchable due to unique IDs +m4_define([AT_BESCMD_DMRPP_RESPONSE_TEST], +[AT_SETUP([BESCMD $1]) +AT_KEYWORDS([dmrpp]) + + input=$abs_srcdir/$1 + baseline=$abs_srcdir/$1.baseline + baseline_alt=$abs_srcdir/$1.baseline_alt + pass=$2 + + AS_IF([test -n "$baselines" -a x$baselines = xyes], + [ + AT_CHECK([bescmdln -i $input], [], [stdout]) + AT_CHECK([mv stdout $baseline.tmp]) + ], + [ + AT_CHECK([bescmdln -i $input], [], [stdout]) + AS_IF(diff -b -B $baseline stdout, + [AT_CHECK([diff -b -B $baseline stdout])], + [ + AT_CHECK([cat stdout | sed 's/.*//g' | sed 's/.*//g' | sed 's/ EffectiveUrlCache::get_effective_url(shared_ptr // It not found or expired, (re)load. if (retrieve_and_cache) { BESDEBUG(MODULE, prolog << "Acquiring effective URL for " << source_url->str() << endl); + INFO_LOG(prolog + "DEPRECATION WARNING: Acquiring effective URL via redirects."); { BES_STOPWATCH_START(MODULE_TIMER, prolog + "Retrieve and cache effective url for source url: " + source_url->str()); try { diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index e384298f7a..822f0d888a 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -44,6 +44,7 @@ #include "CurlUtils.h" #include "CurlHandlePool.h" #include "EffectiveUrlCache.h" +#include "SignedUrlCache.h" #include "DmrppRequestHandler.h" #include "DmrppNames.h" #include "byteswap_compat.h" @@ -51,6 +52,7 @@ using namespace std; using http::EffectiveUrlCache; +using bes::SignedUrlCache; #define prolog std::string("Chunk::").append(__func__).append("() - ") @@ -1348,7 +1350,8 @@ string Chunk::to_string() const { * This method returns the data URL for this chunk. If the data URL is not * set, it returns nullptr. * - * @note The call to get_effective_url() will call EffectiveUrlCache::get_effective_url() + * @note The call to get_signed_url() will first attempt to create a locally-signed url via SignedUrlCache::; if + * that fails, it will fall through to calling EffectiveUrlCache::get_signed_url() * which will call CurlUtils.cc get_redirect_url() which will call gru_mk_attempt() and * will look for an HTTP 302 response and return the redirect URL in that response. * @@ -1359,14 +1362,19 @@ std::shared_ptr Chunk::get_data_url() const { // The d_data_url may be nullptr(fillvalue case). if (d_data_url == nullptr) return d_data_url; - std::shared_ptr effective_url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); - BESDEBUG(MODULE, prolog << "Using data_url: " << effective_url->str() << endl); + + std::shared_ptr url = SignedUrlCache::TheCache()->get_signed_url(d_data_url); + + if (url == nullptr) { + url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); + } + BESDEBUG(MODULE, prolog << "Using data_url: " << url->str() << endl); #if ENABLE_TRACKING_QUERY_PARAMETER //A conditional call to void Chunk::add_tracking_query_param() // here for the NASA cost model work THG's doing. jhrg 8/7/18 if (!d_query_marker.empty()) { - string url_str = effective_url->str(); + string url_str = url->str(); if(url_str.find('?') != string::npos){ url_str.append("&"); } @@ -1379,7 +1387,7 @@ std::shared_ptr Chunk::get_data_url() const { } #endif - return effective_url; + return url; } } // namespace dmrpp diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 4ee7d73031..c881f06f51 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -4,7 +4,8 @@ AUTOMAKE_OPTIONS = foreign subdir-objects ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ - -I$(top_srcdir)/http -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) + -I$(top_srcdir)/http -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) \ + -I$(top_srcdir)/aws AM_CXXFLAGS = -Wno-vla-extension -Wno-inconsistent-missing-override # FIXME Remove this hack. Set these with configure. jhrg 11/25/19 @@ -27,9 +28,23 @@ AM_CPPFLAGS += -DMODULE_NAME=\"$(M_NAME)\" -DMODULE_VERSION=\"$(M_VER)\" # the --disable-shared is not required, but it seems to help with debuggers. CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) + +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + SUBDIRS = ngap_container dmrpp_transmitter build_dmrpp_h4 . unit-tests tests data tests_build_dmrpp get_dmrpp lib_besdir=$(libdir)/bes @@ -54,9 +69,9 @@ vlsa_util.h float_byteswap.h DMRPP_MODULE = DmrppModule.cc DmrppRequestHandler.cc DmrppModule.h DmrppRequestHandler.h libdmrpp_module_la_SOURCES = $(BES_HDRS) $(BES_SRCS) $(DMRPP_MODULE) -libdmrpp_module_la_LDFLAGS = -avoid-version -module +libdmrpp_module_la_LDFLAGS = -avoid-version -module $(AM_LDFLAGS) libdmrpp_module_la_LIBADD = -L$(builddir)/ngap_container -lngap $(BES_DISPATCH_LIB) \ - $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ + $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(LDADD) \ $(H5_LDFLAGS) $(H5_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) -ltest-types \ -Ldmrpp_transmitter -ldmrpp_return_as @@ -76,7 +91,7 @@ build_dmrpp_SOURCES = $(BES_SRCS) $(BES_HDRS) DmrppRequestHandler.cc DmrppReques build_dmrpp_LDFLAGS = $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(H5_LDFLAGS) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) \ - $(XML2_LIBS) $(BYTESWAP_LIBS) -lz + $(LDADD) $(XML2_LIBS) $(BYTESWAP_LIBS) -lz # jhrg 6/2/23 $(BES_EXTRA_LIBS) diff --git a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am index 6b52380ee3..3574d528d4 100644 --- a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am +++ b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am @@ -14,14 +14,28 @@ M_VER=0.1.0 AM_CPPFLAGS = $(HDF4_CFLAGS) $(HDFEOS2_CPPFLAGS) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/modules/dmrpp_module \ -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/http -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/dap \ - $(DAP_CFLAGS) + $(DAP_CFLAGS) -I$(top_srcdir)/aws AM_CPPFLAGS += -DMODULE_NAME=\"$(M_NAME)\" -DMODULE_VERSION=\"$(M_VER)\" AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) + # SUBDIRS = . unit-tests tests # These tests will fail if the DmrApiTest also fails: tests jhrg 6/29/23 @@ -46,12 +60,12 @@ HDR = build_dmrpp_util_h4.h ../Chunk.h ../DMRpp.h ../DMZ.h ../DmrppArray.h ../Dm build_dmrpp_h4_CPPFLAGS = $(AM_CPPFLAGS) #build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDFEOS2_LIBS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAGS) -build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAGS) +build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAGS) $(AM_LDFLAGS) # jhrg 12/18/23 $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_h4_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ - $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) + $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) $(LDADD) lib_besdir=$(libdir)/bes diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index ecd6d3c059..dc2080c185 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -7,7 +7,7 @@ AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ --I$(top_srcdir)/http $(DAP_CFLAGS) +-I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) LIBADD = diff --git a/modules/dmrpp_module/ngap_container/NgapApi.cc b/modules/dmrpp_module/ngap_container/NgapApi.cc index 153a95b2d4..d1ecf5044d 100644 --- a/modules/dmrpp_module/ngap_container/NgapApi.cc +++ b/modules/dmrpp_module/ngap_container/NgapApi.cc @@ -507,7 +507,7 @@ NgapApi::DataAccessUrls NgapApi::convert_ngap_resty_path_to_data_access_urls(con if (data_s3_url.empty() || s3credentials_url.empty()) { // Eventually we'll be removing the non-s3 access; we need to know about any unsupported cases before that happens. // Add a log warning that can be searched. - BES_PROFILE_TIMING(string("PRE-DEPRECATION WARNING - Data s3 url or s3credentials not found - ") + cmr_query_url); + INFO_LOG(prolog + string("PRE-DEPRECATION WARNING - Data s3 url or s3credentials not found - ") + cmr_query_url); } // Check for existing .dmrpp and remove it if found at the end of the url. - kln 6/6/25 diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index 5b90362860..37f45485a9 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -45,10 +45,16 @@ #include "TheBESKeys.h" #include "BESSyntaxUserError.h" #include "BESDebug.h" +#include "EffectiveUrlCache.h" +#include "SignedUrlCache.h" #include "NgapOwnedContainer.h" #include "NgapNames.h" +#define PUGIXML_NO_XPATH +#define PUGIXML_HEADER_ONLY +#include + #define prolog std::string("NgapOwnedContainer::").append(__func__).append("() - ") // CACHE_LOG is defined separately from INFO_LOG so that we can turn it off easily. jhrg 11/19/23 @@ -61,6 +67,9 @@ using namespace std; using namespace bes; +using http::EffectiveUrlCache; +using bes::SignedUrlCache; +using namespace pugi; namespace ngap { @@ -559,6 +568,58 @@ bool NgapOwnedContainer::get_dmrpp_from_cache_or_remote_source(string &dmrpp_str return true; } +static inline bool is_eq(const char *value, const char *key) { + return strcmp(value, key) == 0; +} + +/** + * @brief parse a DMR++ to retrieve the tuple of urls required for NGAP's + * S3 data access. + * + * Implementation adapted from `DMZ::process_dataset`. + * @param dmrpp_string DMR++ + */ +NgapApi::DataAccessUrls NgapOwnedContainer::extract_s3_data_urls_from_dmrpp(const string &dmrpp_string) { + // If the dmrpp is invalid, we will have hit a failure before now---so we can assume + // it's generally safe---so do basically no additional safety checking + string href_attr; + string s3_attr; + string s3credentials_attr; + + // Load the xml document + pugi::xml_document result_xml_doc; + pugi::xml_parse_result result = result_xml_doc.load_string(dmrpp_string.c_str(), pugi::parse_default | pugi::parse_ws_pcdata_single); + if (!result) { + // It would be SO surprising to end up here! Nonetheless, if we do, handle it gracefully. + return tie(href_attr, s3_attr, s3credentials_attr); + } + auto xml_root_node = result_xml_doc.first_child(); + + // Pull the expected data values from the xml + for (xml_attribute attr = xml_root_node.first_attribute(); attr; attr = attr.next_attribute()) { + if (is_eq(attr.name(), "dmrpp:href")) { + href_attr = attr.value(); + } + else if (is_eq(attr.name(), "dmrpp:s3")) { + s3_attr = attr.value(); + } + else if (is_eq(attr.name(), "dmrpp:s3credentials")) { + s3credentials_attr = attr.value(); + } + if (!href_attr.empty() && !s3_attr.empty() && !s3credentials_attr.empty()) { + break; + } + } + + if (s3_attr.empty()) { + BESDEBUG(MODULE, prolog << "DMR++ XML dataset element dmrpp:s3 is missing" << endl); + } + if (s3_attr.empty()) { + BESDEBUG(MODULE, prolog << "DMR++ XML dataset element dmrpp:s3credentials is missing" << endl); + } + return tie(href_attr, s3_attr, s3credentials_attr); +} + /** * @brief Get the DMR++ from a remote source or a local cache * @@ -578,6 +639,11 @@ string NgapOwnedContainer::access() { // get the remote DMR++. jhrg 4/29/24 get_dmrpp_from_cache_or_remote_source(dmrpp_string); + // To sign urls locally, we need access to the credential info that has been previously + // injected into the dmrpp. Extract that now, in preparation for upcoming url signing. + auto urls = extract_s3_data_urls_from_dmrpp(dmrpp_string); + SignedUrlCache::TheCache()->cache_signed_url_components(get<0>(urls), get<1>(urls), get<2>(urls)); + set_attributes("as-string"); // This means access() returns a string. jhrg 10/19/23 // Originally, this was either hard-coded (as it is now) or was set using the 'extension' // on the URL. But it's always a DMR++. jhrg 11/16/23 diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h index 3be72acd60..115c18f1e6 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h @@ -87,6 +87,7 @@ class NgapOwnedContainer: public BESContainer { static FileCache d_dmrpp_file_cache; bool get_dmrpp_from_cache_or_remote_source(std::string &dmrpp_string) const; + static NgapApi::DataAccessUrls extract_s3_data_urls_from_dmrpp(const std::string &dmrpp_string); // I made these statics so that they will be in the class' namespace but still // easy to test in the unit tests. jhrg 4/29/24 diff --git a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am index 29673eed03..889b75f23b 100644 --- a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am +++ b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am @@ -3,7 +3,7 @@ AUTOMAKE_OPTIONS = foreign AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/modules/common \ - -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/dispatch \ + -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/aws -I$(top_srcdir)/dispatch \ -I$(top_srcdir)/dap -I$(top_srcdir)/http $(DAP_CFLAGS) LIBADD = -L$(top_builddir)/modules/common -lmodules_common -L$(builddir)/../ -lngap \ @@ -22,9 +22,24 @@ endif CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align -Werror AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +LDADD_AWS = $(top_builddir)/aws/libbes_aws.la $(aws_libs) + +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + + DISTCLEANFILES = test_config.h *.Po CLEANFILES = *.dbg *.log @@ -103,8 +118,8 @@ MemoryCacheTest_SOURCES = MemoryCacheTest.cc MemoryCacheTest_LDADD = $(LIBADD) NgapApiTest_SOURCES = NgapApiTest.cc -NgapApiTest_LDADD = $(LIBADD) +NgapApiTest_LDADD = $(LIBADD) $(LDADD_AWS) NgapOwnedContainerTest_SOURCES = NgapOwnedContainerTest.cc -NgapOwnedContainerTest_LDADD = $(LIBADD) +NgapOwnedContainerTest_LDADD = $(LIBADD) $(LDADD_AWS) diff --git a/modules/dmrpp_module/unit-tests/Makefile.am b/modules/dmrpp_module/unit-tests/Makefile.am index b9cad3c4af..e8481eb55e 100644 --- a/modules/dmrpp_module/unit-tests/Makefile.am +++ b/modules/dmrpp_module/unit-tests/Makefile.am @@ -10,11 +10,12 @@ AUTOMAKE_OPTIONS = foreign subdir-objects AM_CPPFLAGS = $(H5_CPPFLAGS) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/modules/common \ - -I$(top_srcdir)/modules/dmrpp_module $(DAP_CFLAGS) + -I$(top_srcdir)/modules/dmrpp_module -I$(top_srcdir)/aws $(DAP_CFLAGS) # Added -lz for ubuntu LIBADD = $(BES_DISPATCH_LIB) $(top_builddir)/dap/.libs/libdap_module.a $(BES_HTTP_LIB) \ -L$(top_builddir)/modules/common -lmodules_common $(H5_LDFLAGS) \ + $(top_builddir)/aws/libbes_aws.la $(aws_libs) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LIBS) $(XML2_LIBS) -lz # jhrg 6/2/23 $(BES_EXTRA_LIBS) @@ -29,9 +30,21 @@ endif CXXFLAGS_DEBUG = -g3 -O0 -Wall -Wcast-align AM_CXXFLAGS = -Wno-vla-extension -Wno-inconsistent-missing-override -Wno-unused-variable -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + # This determines what gets built by make check check_PROGRAMS = $(UNIT_TESTS)