Use curl multi interface

CMakeLists.txt

@@ -7,17 +7,7 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-
-find_package(CURL)
-if (CURL_FOUND)
-    message("Found curl")
-    find_package(Threads REQUIRED)
-    include_directories(${CURL_INCLUDE_DIRS})
-    set(ARBITER_CURL TRUE)
-    add_definitions("-DARBITER_CURL")
-else()
-    message("Curl NOT found")
-endif()
+find_package(CURL REQUIRED)
 
 find_package(OpenSSL 1.0)
 if (OPENSSL_FOUND)
@@ -31,11 +21,12 @@ endif()
 
 MESSAGE(${CMAKE_CXX_COMPILER_ID})
 if (${CMAKE_CXX_COMPILER_ID} STREQUAL GNU OR
-        ${CMAKE_CXX_COMPILER_ID} STREQUAL Clang)
+        ${CMAKE_CXX_COMPILER_ID} MATCHES ".*Clang")
     add_definitions("-DUNIX")
     add_definitions(${CMAKE_CXX_FLAGS} "-Wno-deprecated-declarations")
+    add_definitions(${CMAKE_CXX_FLAGS} "-Werror")
     add_definitions(${CMAKE_CXX_FLAGS} "-Wall")
-    add_definitions(${CMAKE_CXX_FLAGS} "-pedantic")
+    add_definitions(${CMAKE_CXX_FLAGS} "-Wextra")
     add_definitions(${CMAKE_CXX_FLAGS} "-pthread")
     add_definitions(${CMAKE_CXX_FLAGS} "-fexceptions")
     add_definitions(${CMAKE_CXX_FLAGS} "-fPIC")
@@ -47,8 +38,6 @@ endif()
 
 include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
 
-
-
 add_subdirectory(arbiter)
 
 set(OBJS
@@ -63,10 +52,8 @@ if (${ARBITER_ZLIB})
     target_include_directories(arbiter PRIVATE "${ZLIB_INCLUDE_DIR}")
 endif()
 
-if (${ARBITER_CURL})
-	target_link_libraries(arbiter PUBLIC ${CURL_LIBRARIES})
-    target_include_directories(arbiter PRIVATE "${CURL_INCLUDE_DIR}")
-endif()
+target_link_libraries(arbiter PUBLIC ${CURL_LIBRARIES})
+target_include_directories(arbiter PRIVATE ${CURL_INCLUDE_DIR})
 
 if (${ARBITER_OPENSSL})
     target_link_libraries(arbiter PUBLIC ${OPENSSL_LIBRARIES})
@@ -77,6 +64,11 @@ target_link_libraries(arbiter PUBLIC ${SHLWAPI})
 
 include (${CMAKE_SOURCE_DIR}/cmake/gtest.cmake)
 
+add_executable(arb
+    cmdline/cmdline.cpp)
+target_link_libraries(arb PRIVATE arbiter)
+target_include_directories(arb PRIVATE arbiter)
+
 add_subdirectory(test)
 
 install(TARGETS arbiter DESTINATION lib)

README.md

@@ -95,5 +95,5 @@ Once the amalgamated files are integrated with your source tree, simply `#includ
 
 Arbiter depends on [Curl](http://curl.haxx.se/libcurl/), which comes preinstalled on most UNIX-based machines.  To manually link (for amalgamated usage) on Unix-based operating systems, link with `-lcurl`.  Arbiter also works on Windows, but you'll have to obtain Curl yourself there.
 
-Arbiter requires C++11.
+Arbiter requires C++17.
 

amalgamate.py

@@ -102,12 +102,6 @@ def amalgamate_source(source_top_dir=None,
         header.add_file("arbiter/third/xml/rapidxml.hpp")
         header.add_file("arbiter/third/xml/xml.hpp")
 
-    if define_curl:
-        header.add_text("\n#pragma once")
-        header.add_text("#define ARBITER_CURL")
-    else:
-        print("NOT #defining ARBITER_CURL")
-
     if bundle_json:
         header.add_file("arbiter/third/json/json.hpp")
     else:

arbiter/arbiter.cpp

@@ -24,10 +24,8 @@ namespace
 {
     const std::string delimiter("://");
 
-#ifdef ARBITER_CURL
     const std::size_t concurrentHttpReqs(32);
     const std::size_t httpRetryCount(8);
-#endif
 
     json getConfig(const std::string& s)
     {
@@ -52,14 +50,12 @@ Arbiter::Arbiter() : Arbiter("") { }
 
 Arbiter::Arbiter(const std::string s)
     : m_config(s)
-#ifdef ARBITER_CURL
     , m_pool(
             new http::Pool(
                 concurrentHttpReqs,
                 httpRetryCount,
                 getConfig(s).dump()))
-#endif
-{ }
+{}
 
 void Arbiter::addDriver(const std::string type, std::shared_ptr<Driver> driver)
 {
@@ -330,7 +326,7 @@ std::shared_ptr<drivers::Http> Arbiter::getHttpDriver(const std::string path) co
 
 LocalHandle Arbiter::getLocalHandle(
         const std::string path,
-        const Endpoint& tempEndpoint) const
+        const Endpoint& /*tempEndpoint*/) const
 {
     const Endpoint fromEndpoint(getEndpoint(getDirname(path)));
     return fromEndpoint.getLocalHandle(getBasename(path));

arbiter/driver.cpp

@@ -29,15 +29,13 @@ std::shared_ptr<Driver> Driver::create(
 
     if (type == "file") return Fs::create();
     if (type == "test") return Test::create();
-#ifdef ARBITER_CURL
     if (type == "http") return Http::create(pool);
     if (type == "https") return Https::create(pool);
     if (type == "s3") return S3::create(pool, entry.dump(), profile);
     if (type == "az") return AZ::create(pool, entry.dump(), profile);
     if (type == "dbx") return Dropbox::create(pool, entry.dump(), profile);
 #ifdef ARBITER_OPENSSL
     if (type == "gs") return Google::create(pool, entry.dump(), profile);
-#endif
 #endif
     return std::shared_ptr<Driver>();
 }
@@ -123,7 +121,7 @@ std::vector<std::string> Driver::resolve(
     return results;
 }
 
-std::vector<std::string> Driver::glob(std::string path, bool verbose) const
+std::vector<std::string> Driver::glob(std::string path, bool /*verbose*/) const
 {
     throw ArbiterError("Cannot glob driver for: " + path);
 }

arbiter/drivers/az.cpp

@@ -46,15 +46,18 @@ namespace
 
     std::string makeLower(const std::string& in)
     {
-        return std::accumulate(
-                in.begin(),
-                in.end(),
-                std::string(),
-                [](const std::string& out, const char c) -> std::string
-                {
-                    return out + static_cast<char>(::tolower(c));
-                });
+        std::string out;
+        for (char c : in)
+            out += (char)std::tolower(c);
+        return out;
+    }
+
+    std::string extractBaseUrl(const std::string& service, const std::string& endpoint,
+        const std::string& account)
+    {
+        return account + "." + service + "." + endpoint + "/";
     }
+
 }
 
 namespace drivers
@@ -88,7 +91,7 @@ AZ::Config::Config(const std::string s)
     , m_storageAccount(extractStorageAccount(s))
     , m_storageAccessKey(extractStorageAccessKey(s))
     , m_endpoint(extractEndpoint(s))
-    , m_baseUrl(extractBaseUrl(s, m_service, m_endpoint, m_storageAccount))
+    , m_baseUrl(extractBaseUrl(m_service, m_endpoint, m_storageAccount))
 {
     const std::string sasString = extractSasToken(s);
     if (!sasString.empty())
@@ -247,18 +250,9 @@ std::string AZ::Config::extractEndpoint(const std::string s)
     return "core.windows.net";
 }
 
-std::string AZ::Config::extractBaseUrl(
-     const std::string s,
-     const std::string service,
-     const std::string endpoint,
-     const std::string account)
-{
-    return account + "." + service + "." + endpoint + "/";
-}
-
 std::unique_ptr<std::size_t> AZ::tryGetSize(
     const std::string rawPath,
-    const http::Headers userHeaders,
+    const http::Headers /*userHeaders*/,
     const http::Query query) const
 {
     Headers headers(m_config->baseHeaders());

arbiter/drivers/az.hpp

@@ -111,7 +111,6 @@ class AZ::Config
 private:
     static std::string extractService(std::string j);
     static std::string extractEndpoint(std::string j);
-    static std::string extractBaseUrl(std::string j, std::string endpoint, std::string service, std::string account);
     static std::string extractStorageAccount(std::string j);
     static std::string extractStorageAccessKey(std::string j);
     static std::string extractSasToken(std::string j);

arbiter/drivers/dropbox.cpp

@@ -147,20 +147,21 @@ bool Dropbox::get(
     headers["Dropbox-API-Arg"] = json{{ "path", "/" + path }}.dump();
     headers.insert(userHeaders.begin(), userHeaders.end());
 
-    const Response res(Http::internalGet(getUrl, headers, query));
+    Response res(Http::internalGet(getUrl, headers, query));
 
     if (res.ok())
     {
         if (!userHeaders.count("Range"))
         {
-            if (!res.headers().count("dropbox-api-result"))
+            Headers headers = res.headers();
+            if (!headers.count("dropbox-api-result"))
             {
                 std::cout << "No dropbox-api-result header found" << std::endl;
                 return false;
             }
 
             json rx;
-            try { rx = json::parse(res.headers().at("dropbox-api-result")); }
+            try { rx = json::parse(headers.at("dropbox-api-result")); }
             catch (...) { std::cout << "Failed to parse result" << std::endl; }
 
             if (!rx.is_null())
@@ -215,9 +216,10 @@ std::vector<char> Dropbox::put(
 
     headers.insert(userHeaders.begin(), userHeaders.end());
 
-    const Response res(Http::internalPost(putUrl, data, headers, query));
+    Response res(Http::internalPost(putUrl, data, headers, query));
 
-    if (!res.ok()) throw ArbiterError(res.str());
+    if (!res.ok())
+        throw ArbiterError(res.str());
     return res.data();
 }
 

arbiter/drivers/fs.cpp

@@ -148,7 +148,7 @@ void Fs::copy(std::string src, std::string dst) const
     outstream << instream.rdbuf();
 }
 
-std::vector<std::string> Fs::glob(std::string path, bool verbose) const
+std::vector<std::string> Fs::glob(std::string path, bool /*verbose*/) const
 {
     return arbiter::glob(path);
 }

arbiter/drivers/google.cpp

@@ -116,8 +116,7 @@ std::unique_ptr<std::size_t> Google::tryGetSize(const std::string path) const
     const GResource resource(path);
 
     drivers::Https https(m_pool);
-    const auto res(
-            https.internalHead(resource.endpoint(), headers, altMediaQuery));
+    http::Response res(https.internalHead(resource.endpoint(), headers, altMediaQuery));
 
     if (res.ok())
     {
@@ -132,14 +131,14 @@ bool Google::get(
         const std::string path,
         std::vector<char>& data,
         const http::Headers userHeaders,
-        const http::Query query) const
+        const http::Query /*query*/) const
 {
     http::Headers headers(m_auth->headers());
     headers.insert(userHeaders.begin(), userHeaders.end());
     const GResource resource(path);
 
     drivers::Https https(m_pool);
-    const auto res(
+    http::Response res(
             https.internalGet(resource.endpoint(), headers, altMediaQuery));
 
     if (res.ok())
@@ -173,12 +172,13 @@ std::vector<char> Google::put(
     query["name"] = http::sanitize(resource.object(), GResource::exclusions);
 
     drivers::Https https(m_pool);
-    const auto res(https.internalPost(url, data, headers, query));
-    if (!res.ok()) throw ArbiterError(res.str());
+    http::Response res(https.internalPost(url, data, headers, query));
+    if (!res.ok())
+        throw ArbiterError(res.str());
     return res.data();
 }
 
-std::vector<std::string> Google::glob(std::string path, bool verbose) const
+std::vector<std::string> Google::glob(std::string path, bool /*verbose*/) const
 {
     std::vector<std::string> results;
 
@@ -203,12 +203,10 @@ std::vector<std::string> Google::glob(std::string path, bool verbose) const
     {
         if (pageToken.size()) query["pageToken"] = pageToken;
 
-        const auto res(https.internalGet(url, m_auth->headers(), query));
+        http::Response res(https.internalGet(url, m_auth->headers(), query));
 
         if (!res.ok())
-        {
             throw ArbiterError(std::to_string(res.code()) + ": " + res.str());
-        }
 
         const json j(json::parse(res.str()));
         for (const json& item : j.at("items"))
@@ -309,7 +307,7 @@ void Google::Auth::maybeRefresh() const
 
     http::Pool pool;
     drivers::Https https(pool);
-    const auto res(https.internalPost(tokenRequestUrl, body, headers));
+    http::Response res(https.internalPost(tokenRequestUrl, body, headers));
 
     if (!res.ok())
     {

arbiter/drivers/http.cpp

@@ -37,11 +37,7 @@ Http::Http(
     : Driver(driverProtocol, profile)
     , m_pool(pool)
     , m_httpProtocol(httpProtocol)
-{
-#ifndef ARBITER_CURL
-    throw ArbiterError("Cannot create HTTP driver - no curl support was built");
-#endif
-}
+{}
 
 std::unique_ptr<Http> Http::create(Pool& pool)
 {
@@ -150,17 +146,11 @@ bool Http::get(
         const Headers headers,
         const Query query) const
 {
-    bool good(false);
-
-    auto http(m_pool.acquire());
+    Resource http(m_pool.acquire());
     Response res(http.get(typedPath(path), headers, query));
 
-
     data = res.data();
-    if (res.ok())
-        good = true;
-
-    return good;
+    return res.ok();
 }
 
 std::vector<char> Http::put(
@@ -169,8 +159,8 @@ std::vector<char> Http::put(
         const Headers headers,
         const Query query) const
 {
-    auto http(m_pool.acquire());
-    auto res(http.put(typedPath(path), data, headers, query));
+    Resource http(m_pool.acquire());
+    Response res(http.put(typedPath(path), data, headers, query));
 
     if (!res.ok())
     {
@@ -195,8 +185,8 @@ void Http::post(
         const Headers headers,
         const Query query) const
 {
-    auto http(m_pool.acquire());
-    auto res(http.post(typedPath(path), data, headers, query));
+    Resource http(m_pool.acquire());
+    Response res(http.post(typedPath(path), data, headers, query));
 
     if (!res.ok())
     {