diff --git a/configure.py b/configure.py index fa47e9c48547f2..9fdb06747316ab 100755 --- a/configure.py +++ b/configure.py @@ -2007,10 +2007,9 @@ def configure_library(lib, output, pkgname=None): output['libraries'] += [pkg_libpath] default_libs = getattr(options, shared_lib + '_libname') - default_libs = [f'-l{l}' for l in default_libs.split(',')] if default_libs: - output['libraries'] += default_libs + output['libraries'] += [f'-l{l}' for l in default_libs.split(',')] elif pkg_libs: output['libraries'] += pkg_libs.split() diff --git a/deps/googletest/include/gtest/internal/gtest-port.h b/deps/googletest/include/gtest/internal/gtest-port.h index 3ea95ba5560714..b607e0adee74f4 100644 --- a/deps/googletest/include/gtest/internal/gtest-port.h +++ b/deps/googletest/include/gtest/internal/gtest-port.h @@ -1236,9 +1236,6 @@ class GTEST_API_ [[nodiscard]] AutoHandle { // Nothing to do here. #else -GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ -/* class A needs to have dll-interface to be used by clients of class B */) - // Allows a controller thread to pause execution of newly created // threads until notified. Instances of this class must be created // and destroyed in the controller thread. @@ -1246,6 +1243,39 @@ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ // This class is only for testing Google Test's own constructs. Do not // use it in user tests, either directly or indirectly. // TODO(b/203539622): Replace unconditionally with absl::Notification. +#ifdef GTEST_OS_WINDOWS_MINGW +// GCC version < 13 with the win32 thread model does not provide std::mutex and +// std::condition_variable in the and headers. So +// we implement the Notification class using a Windows manual-reset event. See +// https://gcc.gnu.org/gcc-13/changes.html#windows. +class GTEST_API_ [[nodiscard]] Notification { + public: + Notification(); + Notification(const Notification&) = delete; + Notification& operator=(const Notification&) = delete; + ~Notification(); + + // Notifies all threads created with this notification to start. Must + // be called from the controller thread. + void Notify(); + + // Blocks until the controller thread notifies. Must be called from a test + // thread. + void WaitForNotification(); + + private: + // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to + // avoid including in this header file. Including is + // undesirable because it defines a lot of symbols and macros that tend to + // conflict with client code. This assumption is verified by + // WindowsTypesTest.HANDLEIsVoidStar. + typedef void* Handle; + Handle event_; +}; +#else +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ +/* class A needs to have dll-interface to be used by clients of class B */) + class GTEST_API_ [[nodiscard]] Notification { public: Notification() : notified_(false) {} @@ -1273,6 +1303,7 @@ class GTEST_API_ [[nodiscard]] Notification { bool notified_; }; GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 +#endif // GTEST_OS_WINDOWS_MINGW #endif // GTEST_HAS_NOTIFICATION_ // On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD diff --git a/deps/googletest/src/gtest-port.cc b/deps/googletest/src/gtest-port.cc index f8ecb37c48d943..d34a693e4500f2 100644 --- a/deps/googletest/src/gtest-port.cc +++ b/deps/googletest/src/gtest-port.cc @@ -303,6 +303,22 @@ bool AutoHandle::IsCloseable() const { return handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE; } +#if !GTEST_HAS_NOTIFICATION_ && defined(GTEST_OS_WINDOWS_MINGW) +Notification::Notification() { + // Create a manual-reset event object. + event_ = ::CreateEvent(nullptr, TRUE, FALSE, nullptr); + GTEST_CHECK_(event_ != nullptr); +} + +Notification::~Notification() { ::CloseHandle(event_); } + +void Notification::Notify() { GTEST_CHECK_(::SetEvent(event_)); } + +void Notification::WaitForNotification() { + GTEST_CHECK_(::WaitForSingleObject(event_, INFINITE) == WAIT_OBJECT_0); +} +#endif // !GTEST_HAS_NOTIFICATION_ && defined(GTEST_OS_WINDOWS_MINGW) + Mutex::Mutex() : owner_thread_id_(0), type_(kDynamic), diff --git a/deps/ncrypto/ncrypto.h b/deps/ncrypto/ncrypto.h index c781d7e2e0288f..4f86702da88267 100644 --- a/deps/ncrypto/ncrypto.h +++ b/deps/ncrypto/ncrypto.h @@ -858,6 +858,9 @@ class EVPKeyPointer final { DER, PEM, JWK, + RAW_PUBLIC, + RAW_PRIVATE, + RAW_SEED, }; enum class PKParseError { NOT_RECOGNIZED, NEED_PASSPHRASE, FAILED }; @@ -867,6 +870,7 @@ class EVPKeyPointer final { bool output_key_object = false; PKFormatType format = PKFormatType::DER; PKEncodingType type = PKEncodingType::PKCS8; + int ec_point_form = POINT_CONVERSION_UNCOMPRESSED; AsymmetricKeyEncodingConfig() = default; AsymmetricKeyEncodingConfig(bool output_key_object, PKFormatType format, diff --git a/deps/ngtcp2/ngtcp2/examples/client.cc b/deps/ngtcp2/ngtcp2/examples/client.cc index c25242962ca44d..56b757b2d91ed1 100644 --- a/deps/ngtcp2/ngtcp2/examples/client.cc +++ b/deps/ngtcp2/ngtcp2/examples/client.cc @@ -1049,7 +1049,7 @@ ngtcp2_ssize write_pkt(ngtcp2_conn *conn, ngtcp2_path *path, ngtcp2_ssize Client::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, uint8_t *dest, size_t destlen, ngtcp2_tstamp ts) { - std::array vec; + std::array vec; for (;;) { int64_t stream_id = -1; @@ -1057,9 +1057,8 @@ ngtcp2_ssize Client::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, nghttp3_ssize sveccnt = 0; if (httpconn_ && ngtcp2_conn_get_max_data_left(conn_)) { - sveccnt = nghttp3_conn_writev_stream( - httpconn_, &stream_id, &fin, - reinterpret_cast(vec.data()), vec.size()); + sveccnt = nghttp3_conn_writev_stream(httpconn_, &stream_id, &fin, + vec.data(), vec.size()); if (sveccnt < 0) { std::cerr << "nghttp3_conn_writev_stream: " << nghttp3_strerror(static_cast(sveccnt)) << std::endl; @@ -1072,6 +1071,7 @@ ngtcp2_ssize Client::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, } ngtcp2_ssize ndatalen; + auto v = vec.data(); auto vcnt = static_cast(sveccnt); uint32_t flags = NGTCP2_WRITE_STREAM_FLAG_MORE; @@ -1081,7 +1081,7 @@ ngtcp2_ssize Client::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, auto nwrite = ngtcp2_conn_writev_stream( conn_, path, pi, dest, destlen, &ndatalen, flags, stream_id, - reinterpret_cast(vec.data()), vcnt, ts); + reinterpret_cast(v), vcnt, ts); if (nwrite < 0) { switch (nwrite) { case NGTCP2_ERR_STREAM_DATA_BLOCKED: diff --git a/deps/ngtcp2/ngtcp2/examples/server.cc b/deps/ngtcp2/ngtcp2/examples/server.cc index 4f081eab920069..f9463eacd9cc7d 100644 --- a/deps/ngtcp2/ngtcp2/examples/server.cc +++ b/deps/ngtcp2/ngtcp2/examples/server.cc @@ -1652,7 +1652,7 @@ ngtcp2_ssize write_pkt(ngtcp2_conn *conn, ngtcp2_path *path, ngtcp2_ssize Handler::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, uint8_t *dest, size_t destlen, ngtcp2_tstamp ts) { - std::array vec; + std::array vec; for (;;) { int64_t stream_id = -1; @@ -1660,9 +1660,8 @@ ngtcp2_ssize Handler::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, nghttp3_ssize sveccnt = 0; if (httpconn_ && ngtcp2_conn_get_max_data_left(conn_)) { - sveccnt = nghttp3_conn_writev_stream( - httpconn_, &stream_id, &fin, - reinterpret_cast(vec.data()), vec.size()); + sveccnt = nghttp3_conn_writev_stream(httpconn_, &stream_id, &fin, + vec.data(), vec.size()); if (sveccnt < 0) { std::cerr << "nghttp3_conn_writev_stream: " << nghttp3_strerror(static_cast(sveccnt)) << std::endl; @@ -1675,6 +1674,7 @@ ngtcp2_ssize Handler::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, } ngtcp2_ssize ndatalen; + auto v = vec.data(); auto vcnt = static_cast(sveccnt); uint32_t flags = @@ -1685,7 +1685,7 @@ ngtcp2_ssize Handler::write_pkt(ngtcp2_path *path, ngtcp2_pkt_info *pi, auto nwrite = ngtcp2_conn_writev_stream( conn_, path, pi, dest, destlen, &ndatalen, flags, stream_id, - reinterpret_cast(vec.data()), vcnt, ts); + reinterpret_cast(v), vcnt, ts); if (nwrite < 0) { switch (nwrite) { case NGTCP2_ERR_STREAM_DATA_BLOCKED: diff --git a/deps/ngtcp2/ngtcp2/examples/shared.h b/deps/ngtcp2/ngtcp2/examples/shared.h index 5455a529a6ee0f..26e678816e8f98 100644 --- a/deps/ngtcp2/ngtcp2/examples/shared.h +++ b/deps/ngtcp2/ngtcp2/examples/shared.h @@ -34,7 +34,6 @@ #include #include -#include #include "network.h" @@ -64,11 +63,6 @@ inline constexpr uint32_t TLS_ALERT_ECH_REQUIRED = 121; inline constexpr size_t MAX_RECV_PKTS = 64; -union SharedVec { - ngtcp2_vec v2; - nghttp3_vec v3; -}; - // msghdr_get_ecn gets ECN bits from |msg|. |family| is the address // family from which packet is received. uint8_t msghdr_get_ecn(msghdr *msg, int family); diff --git a/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/ngtcp2.h b/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/ngtcp2.h index a09ceea382bab1..ba3f7a7e1ae215 100644 --- a/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/ngtcp2.h +++ b/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/ngtcp2.h @@ -1281,7 +1281,7 @@ typedef struct ngtcp2_sockaddr_in { } ngtcp2_sockaddr_in; typedef struct ngtcp2_in6_addr { - uint8_t in6_addr[16]; + uint8_t s6_addr[16]; } ngtcp2_in6_addr; typedef struct ngtcp2_sockaddr_in6 { @@ -1866,8 +1866,8 @@ typedef struct ngtcp2_settings { uint64_t max_stream_window; /** * :member:`ack_thresh` is the minimum number of the received ACK - * eliciting packets that trigger the immediate acknowledgement from - * the local endpoint. + * eliciting packets that triggers the immediate acknowledgement + * from the local endpoint. */ size_t ack_thresh; /** diff --git a/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/version.h b/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/version.h index 6a1ca013450d24..ff43c9c8f8c7c0 100644 --- a/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/version.h +++ b/deps/ngtcp2/ngtcp2/lib/includes/ngtcp2/version.h @@ -36,7 +36,7 @@ * * Version number of the ngtcp2 library release. */ -#define NGTCP2_VERSION "1.20.0" +#define NGTCP2_VERSION "1.21.0" /** * @macro @@ -46,6 +46,6 @@ * number, 8 bits for minor and 8 bits for patch. Version 1.2.3 * becomes 0x010203. */ -#define NGTCP2_VERSION_NUM 0x011400 +#define NGTCP2_VERSION_NUM 0x011500 #endif /* !defined(NGTCP2_VERSION_H) */ diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.c b/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.c index 02865e2a2fc5f6..3db6a9f2bff088 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.c +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.c @@ -76,13 +76,15 @@ static void bbr_init_round_counting(ngtcp2_cc_bbr *bbr); static void bbr_reset_full_bw(ngtcp2_cc_bbr *bbr); -static void bbr_init_pacing_rate(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_init_pacing_rate(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat); -static void bbr_set_pacing_rate_with_gain(ngtcp2_cc_bbr *bbr, +static void bbr_set_pacing_rate_with_gain(const ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, uint64_t pacing_gain_h); -static void bbr_set_pacing_rate(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_set_pacing_rate(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat); static void bbr_enter_startup(ngtcp2_cc_bbr *bbr); @@ -100,47 +102,47 @@ static void bbr_update_control_parameters(ngtcp2_cc_bbr *cc, ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack); -static void bbr_update_on_loss(ngtcp2_cc_bbr *cc, ngtcp2_conn_stat *cstat, - const ngtcp2_cc_pkt *pkt, ngtcp2_tstamp ts); - static void bbr_update_latest_delivery_signals(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); + const ngtcp2_conn_stat *cstat); static void bbr_advance_latest_delivery_signals(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); + const ngtcp2_conn_stat *cstat); static void bbr_update_congestion_signals(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack); -static void bbr_adapt_lower_bounds_from_congestion(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); +static void +bbr_adapt_lower_bounds_from_congestion(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); -static void bbr_init_lower_bounds(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_init_lower_bounds(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); static void bbr_loss_lower_bounds(ngtcp2_cc_bbr *bbr); static void bbr_bound_bw_for_model(ngtcp2_cc_bbr *bbr); -static void bbr_update_max_bw(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static void bbr_update_max_bw(ngtcp2_cc_bbr *bbr, const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack); static void bbr_update_round(ngtcp2_cc_bbr *bbr, const ngtcp2_cc_ack *ack); static void bbr_start_round(ngtcp2_cc_bbr *bbr); -static int bbr_is_in_probe_bw_state(ngtcp2_cc_bbr *bbr); +static int bbr_is_in_probe_bw_state(const ngtcp2_cc_bbr *bbr); -static int bbr_is_probing_bw(ngtcp2_cc_bbr *bbr); +static int bbr_is_probing_bw(const ngtcp2_cc_bbr *bbr); static void bbr_update_ack_aggregation(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack, ngtcp2_tstamp ts); static void bbr_enter_drain(ngtcp2_cc_bbr *bbr); -static void bbr_check_drain_done(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static void bbr_check_drain_done(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat, ngtcp2_tstamp ts); static void bbr_enter_probe_bw(ngtcp2_cc_bbr *bbr, ngtcp2_tstamp ts); @@ -151,57 +153,68 @@ static void bbr_start_probe_bw_cruise(ngtcp2_cc_bbr *bbr); static void bbr_start_probe_bw_refill(ngtcp2_cc_bbr *bbr); -static void bbr_start_probe_bw_up(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_start_probe_bw_up(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); static void bbr_update_probe_bw_cycle_phase(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack, ngtcp2_tstamp ts); -static int bbr_is_time_to_cruise(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static int bbr_is_time_to_cruise(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); -static int bbr_is_time_to_go_down(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static int bbr_is_time_to_go_down(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); -static int bbr_has_elapsed_in_phase(ngtcp2_cc_bbr *bbr, +static int bbr_has_elapsed_in_phase(const ngtcp2_cc_bbr *bbr, ngtcp2_duration interval, ngtcp2_tstamp ts); -static uint64_t bbr_inflight_with_headroom(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); +static uint64_t bbr_inflight_with_headroom(const ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); static void bbr_raise_inflight_longterm_slope(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); + const ngtcp2_conn_stat *cstat); static void bbr_probe_inflight_longterm_upward(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack); static void bbr_adapt_longterm_model(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack); -static int bbr_is_time_to_probe_bw(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static int bbr_is_time_to_probe_bw(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat, ngtcp2_tstamp ts); static void bbr_pick_probe_wait(ngtcp2_cc_bbr *bbr); -static int bbr_is_reno_coexistence_probe_time(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); +static int bbr_is_reno_coexistence_probe_time(const ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); -static uint64_t bbr_target_inflight(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); +static uint64_t bbr_target_inflight(const ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); -static int bbr_is_inflight_too_high(ngtcp2_cc_bbr *bbr, const ngtcp2_rs *rs); +static int bbr_is_inflight_too_high(const ngtcp2_cc_bbr *bbr, + const ngtcp2_rs *rs); static void bbr_handle_inflight_too_high(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_rs *rs, ngtcp2_tstamp ts); static void bbr_note_loss(ngtcp2_cc_bbr *bbr); -static void bbr_handle_lost_packet(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static void bbr_save_state_upon_loss(ngtcp2_cc_bbr *bbr); + +static void bbr_handle_spurious_loss_detection(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); + +static void bbr_handle_lost_packet(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_pkt *pkt, ngtcp2_tstamp ts); -static uint64_t bbr_inflight_at_loss(ngtcp2_cc_bbr *bbr, +static uint64_t bbr_inflight_at_loss(const ngtcp2_cc_bbr *bbr, const ngtcp2_cc_pkt *pkt, const ngtcp2_rs *rs); @@ -220,7 +233,7 @@ static void bbr_check_probe_rtt_done(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, ngtcp2_tstamp ts); static void bbr_mark_connection_app_limited(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); + const ngtcp2_conn_stat *cstat); static void bbr_exit_probe_rtt(ngtcp2_cc_bbr *bbr, ngtcp2_tstamp ts); @@ -231,27 +244,28 @@ static void bbr_handle_restart_from_idle(ngtcp2_cc_bbr *bbr, static uint64_t bbr_bdp_multiple(ngtcp2_cc_bbr *bbr, uint64_t gain_h); static uint64_t bbr_quantization_budget(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, uint64_t inflight); -static uint64_t bbr_inflight(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static uint64_t bbr_inflight(ngtcp2_cc_bbr *bbr, const ngtcp2_conn_stat *cstat, uint64_t gain_h); static void bbr_update_max_inflight(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); + const ngtcp2_conn_stat *cstat); static void bbr_update_offload_budget(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat); + const ngtcp2_conn_stat *cstat); static uint64_t min_pipe_cwnd(size_t max_udp_payload_size); static void bbr_advance_max_bw_filter(ngtcp2_cc_bbr *bbr); -static void bbr_save_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_save_cwnd(ngtcp2_cc_bbr *bbr, const ngtcp2_conn_stat *cstat); -static void bbr_restore_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_restore_cwnd(const ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); -static uint64_t bbr_probe_rtt_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static uint64_t bbr_probe_rtt_cwnd(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat); static void bbr_bound_cwnd_for_probe_rtt(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); @@ -259,10 +273,11 @@ static void bbr_bound_cwnd_for_probe_rtt(ngtcp2_cc_bbr *bbr, static void bbr_set_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack); -static void bbr_bound_cwnd_for_model(ngtcp2_cc_bbr *bbr, +static void bbr_bound_cwnd_for_model(const ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); -static void bbr_set_send_quantum(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat); +static void bbr_set_send_quantum(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat); static int in_congestion_recovery(const ngtcp2_conn_stat *cstat, ngtcp2_tstamp sent_time); @@ -336,6 +351,7 @@ static void bbr_on_init(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, bbr->bdp = 0; + bbr->undo_state = 0; bbr->undo_bw_shortterm = 0; bbr->undo_inflight_shortterm = 0; bbr->undo_inflight_longterm = 0; @@ -365,7 +381,7 @@ static void bbr_reset_full_bw(ngtcp2_cc_bbr *bbr) { } static void bbr_check_full_bw_reached(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { if (bbr->full_bw_now || !bbr->round_start || bbr->rst->rs.is_app_limited) { return; } @@ -404,7 +420,8 @@ static void bbr_check_startup_high_loss(ngtcp2_cc_bbr *bbr) { bbr_bdp_multiple(bbr, bbr->cwnd_gain_h), bbr->inflight_latest); } -static void bbr_init_pacing_rate(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_init_pacing_rate(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat) { cstat->pacing_interval_m = ngtcp2_max_uint64( ((cstat->first_rtt_sample_ts == UINT64_MAX ? NGTCP2_MILLISECONDS : cstat->smoothed_rtt) @@ -413,7 +430,7 @@ static void bbr_init_pacing_rate(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { 1); } -static void bbr_set_pacing_rate_with_gain(ngtcp2_cc_bbr *bbr, +static void bbr_set_pacing_rate_with_gain(const ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, uint64_t pacing_gain_h) { uint64_t interval_m; @@ -431,7 +448,8 @@ static void bbr_set_pacing_rate_with_gain(ngtcp2_cc_bbr *bbr, } } -static void bbr_set_pacing_rate(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_set_pacing_rate(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat) { bbr_set_pacing_rate_with_gain(bbr, cstat, bbr->pacing_gain_h); } @@ -487,13 +505,8 @@ static void bbr_update_control_parameters(ngtcp2_cc_bbr *bbr, bbr_set_cwnd(bbr, cstat, ack); } -static void bbr_update_on_loss(ngtcp2_cc_bbr *cc, ngtcp2_conn_stat *cstat, - const ngtcp2_cc_pkt *pkt, ngtcp2_tstamp ts) { - bbr_handle_lost_packet(cc, cstat, pkt, ts); -} - static void bbr_update_latest_delivery_signals(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { bbr->loss_round_start = 0; bbr->bw_latest = ngtcp2_max_uint64(bbr->bw_latest, cstat->delivery_rate_sec); bbr->inflight_latest = @@ -506,7 +519,7 @@ static void bbr_update_latest_delivery_signals(ngtcp2_cc_bbr *bbr, } static void bbr_advance_latest_delivery_signals(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { if (bbr->loss_round_start) { bbr->bw_latest = cstat->delivery_rate_sec; bbr->inflight_latest = bbr->rst->rs.delivered; @@ -514,7 +527,7 @@ static void bbr_advance_latest_delivery_signals(ngtcp2_cc_bbr *bbr, } static void bbr_update_congestion_signals(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack) { bbr_update_max_bw(bbr, cstat, ack); @@ -532,8 +545,9 @@ static void bbr_update_congestion_signals(ngtcp2_cc_bbr *bbr, bbr->loss_in_round = 0; } -static void bbr_adapt_lower_bounds_from_congestion(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { +static void +bbr_adapt_lower_bounds_from_congestion(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { if (bbr_is_probing_bw(bbr)) { return; } @@ -544,7 +558,8 @@ static void bbr_adapt_lower_bounds_from_congestion(ngtcp2_cc_bbr *bbr, } } -static void bbr_init_lower_bounds(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_init_lower_bounds(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { if (bbr->bw_shortterm == UINT64_MAX) { bbr->bw_shortterm = bbr->max_bw; } @@ -567,7 +582,7 @@ static void bbr_bound_bw_for_model(ngtcp2_cc_bbr *bbr) { bbr->bw = ngtcp2_min_uint64(bbr->max_bw, bbr->bw_shortterm); } -static void bbr_update_max_bw(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static void bbr_update_max_bw(ngtcp2_cc_bbr *bbr, const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack) { bbr_update_round(bbr, ack); @@ -603,7 +618,7 @@ static void bbr_start_round(ngtcp2_cc_bbr *bbr) { bbr->next_round_delivered = bbr->rst->delivered; } -static int bbr_is_in_probe_bw_state(ngtcp2_cc_bbr *bbr) { +static int bbr_is_in_probe_bw_state(const ngtcp2_cc_bbr *bbr) { switch (bbr->state) { case NGTCP2_BBR_STATE_PROBE_BW_DOWN: case NGTCP2_BBR_STATE_PROBE_BW_CRUISE: @@ -615,7 +630,7 @@ static int bbr_is_in_probe_bw_state(ngtcp2_cc_bbr *bbr) { } } -static int bbr_is_probing_bw(ngtcp2_cc_bbr *bbr) { +static int bbr_is_probing_bw(const ngtcp2_cc_bbr *bbr) { switch (bbr->state) { case NGTCP2_BBR_STATE_STARTUP: case NGTCP2_BBR_STATE_PROBE_BW_REFILL: @@ -627,7 +642,7 @@ static int bbr_is_probing_bw(ngtcp2_cc_bbr *bbr) { } static void bbr_update_ack_aggregation(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack, ngtcp2_tstamp ts) { ngtcp2_duration interval = ts - bbr->extra_acked_interval_start; @@ -669,7 +684,8 @@ static void bbr_enter_drain(ngtcp2_cc_bbr *bbr) { bbr->cwnd_gain_h = NGTCP2_BBR_DEFAULT_CWND_GAIN_H; } -static void bbr_check_drain_done(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static void bbr_check_drain_done(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat, ngtcp2_tstamp ts) { if (bbr->state == NGTCP2_BBR_STATE_DRAIN && cstat->bytes_in_flight <= bbr_inflight(bbr, cstat, 100)) { @@ -726,7 +742,8 @@ static void bbr_start_probe_bw_refill(ngtcp2_cc_bbr *bbr) { bbr->cwnd_gain_h = NGTCP2_BBR_DEFAULT_CWND_GAIN_H; } -static void bbr_start_probe_bw_up(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_start_probe_bw_up(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { ngtcp2_log_info(bbr->cc.log, NGTCP2_LOG_EVENT_CCA, "bbr start ProbeBW_UP"); bbr->ack_phase = NGTCP2_BBR_ACK_PHASE_ACKS_PROBE_STARTING; @@ -743,7 +760,7 @@ static void bbr_start_probe_bw_up(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { } static void bbr_update_probe_bw_cycle_phase(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack, ngtcp2_tstamp ts) { if (!bbr->full_bw_reached) { @@ -791,14 +808,16 @@ static void bbr_update_probe_bw_cycle_phase(ngtcp2_cc_bbr *bbr, } } -static int bbr_is_time_to_cruise(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static int bbr_is_time_to_cruise(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { uint64_t inflight = ngtcp2_min_uint64(bbr_inflight_with_headroom(bbr, cstat), bbr_inflight(bbr, cstat, 100)); return cstat->bytes_in_flight <= inflight; } -static int bbr_is_time_to_go_down(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static int bbr_is_time_to_go_down(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { if (bbr->rst->is_cwnd_limited && cstat->cwnd >= bbr->inflight_longterm) { bbr_reset_full_bw(bbr); bbr->full_bw = cstat->delivery_rate_sec; @@ -809,14 +828,14 @@ static int bbr_is_time_to_go_down(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { return bbr->full_bw_now; } -static int bbr_has_elapsed_in_phase(ngtcp2_cc_bbr *bbr, +static int bbr_has_elapsed_in_phase(const ngtcp2_cc_bbr *bbr, ngtcp2_duration interval, ngtcp2_tstamp ts) { return ts > bbr->cycle_stamp + interval; } -static uint64_t bbr_inflight_with_headroom(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { +static uint64_t bbr_inflight_with_headroom(const ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { uint64_t headroom; uint64_t mpcwnd; if (bbr->inflight_longterm == UINT64_MAX) { @@ -837,7 +856,7 @@ static uint64_t bbr_inflight_with_headroom(ngtcp2_cc_bbr *bbr, } static void bbr_raise_inflight_longterm_slope(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { uint64_t growth_this_round = cstat->max_tx_udp_payload_size << bbr->bw_probe_up_rounds; @@ -846,7 +865,7 @@ static void bbr_raise_inflight_longterm_slope(ngtcp2_cc_bbr *bbr, } static void bbr_probe_inflight_longterm_upward(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack) { uint64_t delta; @@ -870,7 +889,7 @@ static void bbr_probe_inflight_longterm_upward(ngtcp2_cc_bbr *bbr, } static void bbr_adapt_longterm_model(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_ack *ack) { if (bbr->ack_phase == NGTCP2_BBR_ACK_PHASE_ACKS_PROBE_STARTING && bbr->round_start) { @@ -899,7 +918,8 @@ static void bbr_adapt_longterm_model(ngtcp2_cc_bbr *bbr, } } -static int bbr_is_time_to_probe_bw(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static int bbr_is_time_to_probe_bw(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat, ngtcp2_tstamp ts) { if (bbr_has_elapsed_in_phase(bbr, bbr->bw_probe_wait, ts) || bbr_is_reno_coexistence_probe_time(bbr, cstat)) { @@ -917,27 +937,28 @@ static void bbr_pick_probe_wait(ngtcp2_cc_bbr *bbr) { 2 * NGTCP2_SECONDS + ngtcp2_pcg32_rand_n(bbr->pcg, NGTCP2_SECONDS + 1); } -static int bbr_is_reno_coexistence_probe_time(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { +static int bbr_is_reno_coexistence_probe_time(const ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { uint64_t reno_rounds = bbr_target_inflight(bbr, cstat) / cstat->max_tx_udp_payload_size; return bbr->rounds_since_bw_probe >= ngtcp2_min_uint64(reno_rounds, 63); } -static uint64_t bbr_target_inflight(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { +static uint64_t bbr_target_inflight(const ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { return ngtcp2_min_uint64(bbr->bdp, cstat->cwnd); } -static int bbr_is_inflight_too_high(ngtcp2_cc_bbr *bbr, const ngtcp2_rs *rs) { +static int bbr_is_inflight_too_high(const ngtcp2_cc_bbr *bbr, + const ngtcp2_rs *rs) { (void)bbr; return rs->lost * NGTCP2_BBR_LOSS_THRESH_DENOM > rs->tx_in_flight * NGTCP2_BBR_LOSS_THRESH_NUMER; } static void bbr_handle_inflight_too_high(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, const ngtcp2_rs *rs, ngtcp2_tstamp ts) { bbr->bw_probe_samples = 0; @@ -956,12 +977,49 @@ static void bbr_handle_inflight_too_high(ngtcp2_cc_bbr *bbr, static void bbr_note_loss(ngtcp2_cc_bbr *bbr) { if (!bbr->loss_in_round) { bbr->loss_round_delivered = bbr->rst->delivered; + bbr_save_state_upon_loss(bbr); } bbr->loss_in_round = 1; } -static void bbr_handle_lost_packet(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static void bbr_save_state_upon_loss(ngtcp2_cc_bbr *bbr) { + bbr->undo_state = bbr->state; + bbr->undo_bw_shortterm = bbr->bw_shortterm; + bbr->undo_inflight_shortterm = bbr->inflight_shortterm; + bbr->undo_inflight_longterm = bbr->inflight_longterm; +} + +static void bbr_handle_spurious_loss_detection(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat) { + bbr->loss_in_round = 0; + + bbr_reset_full_bw(bbr); + + bbr->bw_shortterm = + ngtcp2_max_uint64(bbr->bw_shortterm, bbr->undo_bw_shortterm); + bbr->inflight_shortterm = + ngtcp2_max_uint64(bbr->inflight_shortterm, bbr->undo_inflight_shortterm); + bbr->inflight_longterm = + ngtcp2_max_uint64(bbr->inflight_longterm, bbr->undo_inflight_longterm); + + if (bbr->state != NGTCP2_BBR_STATE_PROBE_RTT && + bbr->state != bbr->undo_state) { + switch (bbr->undo_state) { + case NGTCP2_BBR_STATE_STARTUP: + bbr_enter_startup(bbr); + break; + case NGTCP2_BBR_STATE_PROBE_BW_UP: + bbr_start_probe_bw_up(bbr, cstat); + break; + default: + break; + } + } +} + +static void bbr_handle_lost_packet(ngtcp2_cc_bbr *bbr, + const ngtcp2_conn_stat *cstat, const ngtcp2_cc_pkt *pkt, ngtcp2_tstamp ts) { ngtcp2_rs rs = {0}; @@ -983,7 +1041,7 @@ static void bbr_handle_lost_packet(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, } } -static uint64_t bbr_inflight_at_loss(ngtcp2_cc_bbr *bbr, +static uint64_t bbr_inflight_at_loss(const ngtcp2_cc_bbr *bbr, const ngtcp2_cc_pkt *pkt, const ngtcp2_rs *rs) { uint64_t inflight_prev, lost_prev, lost_prefix; @@ -1100,7 +1158,7 @@ static void bbr_check_probe_rtt_done(ngtcp2_cc_bbr *bbr, } static void bbr_mark_connection_app_limited(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { bbr->rst->app_limited = ngtcp2_max_uint64(bbr->rst->delivered + cstat->bytes_in_flight, 1); } @@ -1148,7 +1206,7 @@ static uint64_t min_pipe_cwnd(size_t max_udp_payload_size) { } static uint64_t bbr_quantization_budget(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat, + const ngtcp2_conn_stat *cstat, uint64_t inflight) { bbr_update_offload_budget(bbr, cstat); @@ -1163,7 +1221,7 @@ static uint64_t bbr_quantization_budget(ngtcp2_cc_bbr *bbr, return inflight; } -static uint64_t bbr_inflight(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, +static uint64_t bbr_inflight(ngtcp2_cc_bbr *bbr, const ngtcp2_conn_stat *cstat, uint64_t gain_h) { uint64_t inflight = bbr_bdp_multiple(bbr, gain_h); @@ -1171,7 +1229,7 @@ static uint64_t bbr_inflight(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, } static void bbr_update_max_inflight(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { uint64_t inflight; inflight = bbr_bdp_multiple(bbr, bbr->cwnd_gain_h) + bbr->extra_acked; @@ -1179,7 +1237,7 @@ static void bbr_update_max_inflight(ngtcp2_cc_bbr *bbr, } static void bbr_update_offload_budget(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { bbr->offload_budget = 3 * cstat->send_quantum; } @@ -1187,7 +1245,7 @@ static void bbr_advance_max_bw_filter(ngtcp2_cc_bbr *bbr) { ++bbr->cycle_count; } -static void bbr_save_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_save_cwnd(ngtcp2_cc_bbr *bbr, const ngtcp2_conn_stat *cstat) { if (!bbr->in_loss_recovery && bbr->state != NGTCP2_BBR_STATE_PROBE_RTT) { bbr->prior_cwnd = cstat->cwnd; return; @@ -1196,12 +1254,13 @@ static void bbr_save_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { bbr->prior_cwnd = ngtcp2_max_uint64(bbr->prior_cwnd, cstat->cwnd); } -static void bbr_restore_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_restore_cwnd(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat) { cstat->cwnd = ngtcp2_max_uint64(cstat->cwnd, bbr->prior_cwnd); } static uint64_t bbr_probe_rtt_cwnd(ngtcp2_cc_bbr *bbr, - ngtcp2_conn_stat *cstat) { + const ngtcp2_conn_stat *cstat) { uint64_t probe_rtt_cwnd = bbr_bdp_multiple(bbr, NGTCP2_BBR_PROBE_RTT_CWND_GAIN_H); uint64_t mpcwnd = min_pipe_cwnd(cstat->max_tx_udp_payload_size); @@ -1238,7 +1297,7 @@ static void bbr_set_cwnd(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat, bbr_bound_cwnd_for_model(bbr, cstat); } -static void bbr_bound_cwnd_for_model(ngtcp2_cc_bbr *bbr, +static void bbr_bound_cwnd_for_model(const ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { uint64_t cap = UINT64_MAX; uint64_t mpcwnd = min_pipe_cwnd(cstat->max_tx_udp_payload_size); @@ -1257,7 +1316,8 @@ static void bbr_bound_cwnd_for_model(ngtcp2_cc_bbr *bbr, cstat->cwnd = ngtcp2_min_uint64(cstat->cwnd, cap); } -static void bbr_set_send_quantum(ngtcp2_cc_bbr *bbr, ngtcp2_conn_stat *cstat) { +static void bbr_set_send_quantum(const ngtcp2_cc_bbr *bbr, + ngtcp2_conn_stat *cstat) { size_t send_quantum = 64 * 1024; (void)bbr; @@ -1293,7 +1353,7 @@ static void bbr_cc_on_pkt_lost(ngtcp2_cc *cc, ngtcp2_conn_stat *cstat, const ngtcp2_cc_pkt *pkt, ngtcp2_tstamp ts) { ngtcp2_cc_bbr *bbr = ngtcp2_struct_of(cc, ngtcp2_cc_bbr, cc); - bbr_update_on_loss(bbr, cstat, pkt, ts); + bbr_handle_lost_packet(bbr, cstat, pkt, ts); } static void bbr_cc_congestion_event(ngtcp2_cc *cc, ngtcp2_conn_stat *cstat, @@ -1311,9 +1371,6 @@ static void bbr_cc_congestion_event(ngtcp2_cc *cc, ngtcp2_conn_stat *cstat, bbr->round_count_at_recovery = bbr->round_start ? bbr->round_count : bbr->round_count + 1; bbr_save_cwnd(bbr, cstat); - bbr->undo_bw_shortterm = bbr->bw_shortterm; - bbr->undo_inflight_shortterm = bbr->inflight_shortterm; - bbr->undo_inflight_longterm = bbr->inflight_longterm; cstat->congestion_recovery_start_ts = ts; } @@ -1328,15 +1385,9 @@ static void bbr_cc_on_spurious_congestion(ngtcp2_cc *cc, bbr->in_loss_recovery = 0; bbr->round_count_at_recovery = UINT64_MAX; - bbr_reset_full_bw(bbr); - bbr->loss_in_round = 0; + bbr_restore_cwnd(bbr, cstat); - bbr->bw_shortterm = - ngtcp2_max_uint64(bbr->bw_shortterm, bbr->undo_bw_shortterm); - bbr->inflight_shortterm = - ngtcp2_max_uint64(bbr->inflight_shortterm, bbr->undo_inflight_shortterm); - bbr->inflight_longterm = - ngtcp2_max_uint64(bbr->inflight_longterm, bbr->undo_inflight_longterm); + bbr_handle_spurious_loss_detection(bbr, cstat); } static void bbr_cc_on_persistent_congestion(ngtcp2_cc *cc, diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.h b/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.h index 98b23122ff9908..5177944b290781 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.h +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_bbr.h @@ -107,6 +107,7 @@ typedef struct ngtcp2_cc_bbr { uint64_t cwnd_gain_h; /* Backup for spurious losses */ + ngtcp2_bbr_state undo_state; uint64_t undo_bw_shortterm; uint64_t undo_inflight_shortterm; uint64_t undo_inflight_longterm; diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_conn.c b/deps/ngtcp2/ngtcp2/lib/ngtcp2_conn.c index 84a41943a801d5..29fe7b03911aa3 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_conn.c +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_conn.c @@ -11913,6 +11913,8 @@ conn_write_vmsg_wrapper(ngtcp2_conn *conn, ngtcp2_path *path, return nwrite; } + assert((size_t)nwrite <= destlen); + if (cstat->bytes_in_flight >= cstat->cwnd) { conn->rst.is_cwnd_limited = 1; } else if ((cstat->cwnd >= cstat->ssthresh || @@ -12355,7 +12357,7 @@ ngtcp2_ssize ngtcp2_conn_write_vmsg(ngtcp2_conn *conn, ngtcp2_path *path, /* We only exceed CWND to avoid deadlock. Do no write 1RTT packet if CWND is depleted. */ if (conn_cwnd_is_zero(conn) && conn->pktns.rtb.probe_pkt_left == 0) { - goto fin; + return res; } } else if (destlen == 0) { res = conn_write_handshake_ack_pkts(conn, pi, dest, origlen, ts); diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.c b/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.c index 0c1c0a36c5bcf9..be3c9e182b5d06 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.c +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.c @@ -31,7 +31,6 @@ #include "ngtcp2_macro.h" #include "ngtcp2_mem.h" -#include "ngtcp2_range.h" static ngtcp2_ksl_blk null_blk; @@ -815,12 +814,6 @@ int ngtcp2_ksl_it_begin(const ngtcp2_ksl_it *it) { return it->i == 0 && it->blk->prev == NULL; } -int ngtcp2_ksl_range_compar(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs) { - const ngtcp2_range *a = lhs, *b = rhs; - return a->begin < b->begin; -} - ngtcp2_ksl_search_def(range, ngtcp2_ksl_range_compar) size_t ngtcp2_ksl_range_search(const ngtcp2_ksl *ksl, ngtcp2_ksl_blk *blk, @@ -828,13 +821,6 @@ size_t ngtcp2_ksl_range_search(const ngtcp2_ksl *ksl, ngtcp2_ksl_blk *blk, return ksl_range_search(ksl, blk, key); } -int ngtcp2_ksl_range_exclusive_compar(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs) { - const ngtcp2_range *a = lhs, *b = rhs; - return a->begin < b->begin && !(ngtcp2_max_uint64(a->begin, b->begin) < - ngtcp2_min_uint64(a->end, b->end)); -} - ngtcp2_ksl_search_def(range_exclusive, ngtcp2_ksl_range_exclusive_compar) size_t ngtcp2_ksl_range_exclusive_search(const ngtcp2_ksl *ksl, @@ -843,11 +829,6 @@ size_t ngtcp2_ksl_range_exclusive_search(const ngtcp2_ksl *ksl, return ksl_range_exclusive_search(ksl, blk, key); } -int ngtcp2_ksl_uint64_less(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs) { - return *(uint64_t *)lhs < *(uint64_t *)rhs; -} - ngtcp2_ksl_search_def(uint64_less, ngtcp2_ksl_uint64_less) size_t ngtcp2_ksl_uint64_less_search(const ngtcp2_ksl *ksl, ngtcp2_ksl_blk *blk, @@ -855,11 +836,6 @@ size_t ngtcp2_ksl_uint64_less_search(const ngtcp2_ksl *ksl, ngtcp2_ksl_blk *blk, return ksl_uint64_less_search(ksl, blk, key); } -int ngtcp2_ksl_int64_greater(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs) { - return *(int64_t *)lhs > *(int64_t *)rhs; -} - ngtcp2_ksl_search_def(int64_greater, ngtcp2_ksl_int64_greater) size_t ngtcp2_ksl_int64_greater_search(const ngtcp2_ksl *ksl, diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.h b/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.h index cb972f94dca2e2..8024a360cdb4fc 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.h +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_ksl.h @@ -34,6 +34,7 @@ #include #include "ngtcp2_objalloc.h" +#include "ngtcp2_range.h" #define NGTCP2_KSL_DEGR 16 /* NGTCP2_KSL_MAX_NBLK is the maximum number of nodes which a single @@ -356,8 +357,12 @@ static inline const ngtcp2_ksl_key *ngtcp2_ksl_it_key(const ngtcp2_ksl_it *it) { * returns nonzero if ((const ngtcp2_range *)lhs)->begin < ((const * ngtcp2_range *)rhs)->begin. */ -int ngtcp2_ksl_range_compar(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs); +static inline int ngtcp2_ksl_range_compar(const ngtcp2_ksl_key *lhs, + const ngtcp2_ksl_key *rhs) { + const ngtcp2_range *a = (const ngtcp2_range *)lhs, + *b = (const ngtcp2_range *)rhs; + return a->begin < b->begin; +} /* * ngtcp2_ksl_range_search is an implementation of ngtcp2_ksl_search @@ -373,8 +378,13 @@ size_t ngtcp2_ksl_range_search(const ngtcp2_ksl *ksl, ngtcp2_ksl_blk *blk, * *)lhs)->begin < ((const ngtcp2_range *)rhs)->begin, and the 2 * ranges do not intersect. */ -int ngtcp2_ksl_range_exclusive_compar(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs); +static inline int ngtcp2_ksl_range_exclusive_compar(const ngtcp2_ksl_key *lhs, + const ngtcp2_ksl_key *rhs) { + const ngtcp2_range *a = (const ngtcp2_range *)lhs, + *b = (const ngtcp2_range *)rhs; + return a->begin < b->begin && !(ngtcp2_max_uint64(a->begin, b->begin) < + ngtcp2_min_uint64(a->end, b->end)); +} /* * ngtcp2_ksl_range_exclusive_search is an implementation of @@ -389,8 +399,10 @@ size_t ngtcp2_ksl_range_exclusive_search(const ngtcp2_ksl *ksl, * |lhs| and |rhs| must point to uint64_t objects, and the function * returns nonzero if *(uint64_t *)|lhs| < *(uint64_t *)|rhs|. */ -int ngtcp2_ksl_uint64_less(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs); +static inline int ngtcp2_ksl_uint64_less(const ngtcp2_ksl_key *lhs, + const ngtcp2_ksl_key *rhs) { + return *(const uint64_t *)lhs < *(const uint64_t *)rhs; +} /* * ngtcp2_ksl_uint64_less_search is an implementation of @@ -404,8 +416,10 @@ size_t ngtcp2_ksl_uint64_less_search(const ngtcp2_ksl *ksl, ngtcp2_ksl_blk *blk, * |lhs| and |rhs| must point to int64_t objects, and the function * returns nonzero if *(int64_t *)|lhs| > *(int64_t *)|rhs|. */ -int ngtcp2_ksl_int64_greater(const ngtcp2_ksl_key *lhs, - const ngtcp2_ksl_key *rhs); +static inline int ngtcp2_ksl_int64_greater(const ngtcp2_ksl_key *lhs, + const ngtcp2_ksl_key *rhs) { + return *(const int64_t *)lhs > *(const int64_t *)rhs; +} /* * ngtcp2_ksl_int64_greater_search is an implementation of diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.c b/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.c index 2015e401ef2609..39fd6969a5e3f6 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.c +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.c @@ -146,33 +146,22 @@ static const char *strapperrorcode(uint64_t app_error_code) { return "(unknown)"; } -static const char *strpkttype_long(uint8_t type) { - switch (type) { +static const char *strpkttype(const ngtcp2_pkt_hd *hd) { + switch (hd->type) { case NGTCP2_PKT_INITIAL: return "Initial"; - case NGTCP2_PKT_RETRY: - return "Retry"; - case NGTCP2_PKT_HANDSHAKE: - return "Handshake"; case NGTCP2_PKT_0RTT: return "0RTT"; - default: - return "(unknown)"; - } -} - -static const char *strpkttype(const ngtcp2_pkt_hd *hd) { - if (hd->flags & NGTCP2_PKT_FLAG_LONG_FORM) { - return strpkttype_long(hd->type); - } - - switch (hd->type) { + case NGTCP2_PKT_HANDSHAKE: + return "Handshake"; + case NGTCP2_PKT_RETRY: + return "Retry"; + case NGTCP2_PKT_1RTT: + return "1RTT"; case NGTCP2_PKT_VERSION_NEGOTIATION: return "VN"; case NGTCP2_PKT_STATELESS_RESET: return "SR"; - case NGTCP2_PKT_1RTT: - return "1RTT"; default: return "(unknown)"; } @@ -266,7 +255,7 @@ static void log_fr_connection_close(ngtcp2_log *log, const ngtcp2_pkt_hd *hd, log, NGTCP2_LOG_EVENT_FRM, NGTCP2_LOG_PKT " CONNECTION_CLOSE(0x%02" PRIx64 ") error_code=%s(0x%" PRIx64 ") " - "frame_type=%" PRIx64 " reason_len=%zu reason=[%s]", + "frame_type=0x%" PRIx64 " reason_len=%zu reason=[%s]", NGTCP2_LOG_PKT_HD_FIELDS(dir), fr->type, fr->type == NGTCP2_FRAME_CONNECTION_CLOSE ? strerrorcode(fr->error_code) : strapperrorcode(fr->error_code), @@ -766,12 +755,6 @@ void ngtcp2_log_tx_pkt_hd(ngtcp2_log *log, const ngtcp2_pkt_hd *hd) { log_pkt_hd(log, hd, "tx"); } -void ngtcp2_log_tx_cancel(ngtcp2_log *log, const ngtcp2_pkt_hd *hd) { - ngtcp2_log_infof(log, NGTCP2_LOG_EVENT_PKT, - "cancel tx pkn=%" PRId64 " type=%s", hd->pkt_num, - strpkttype(hd)); -} - uint64_t ngtcp2_log_timestamp(const ngtcp2_log *log) { return (log->last_ts - log->ts) / NGTCP2_MILLISECONDS; } diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.h b/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.h index fabd5908e0d492..02726bc19ded6f 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.h +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_log.h @@ -119,8 +119,6 @@ void ngtcp2_log_rx_pkt_hd(ngtcp2_log *log, const ngtcp2_pkt_hd *hd); void ngtcp2_log_tx_pkt_hd(ngtcp2_log *log, const ngtcp2_pkt_hd *hd); -void ngtcp2_log_tx_cancel(ngtcp2_log *log, const ngtcp2_pkt_hd *hd); - #define NGTCP2_LOG_HD "I%08" PRIu64 " 0x%s %s" uint64_t ngtcp2_log_timestamp(const ngtcp2_log *log); diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.c b/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.c index 0179dbcde96445..a63913f8af797a 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.c +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.c @@ -543,7 +543,7 @@ ngtcp2_ssize ngtcp2_frame_decoder_decode(ngtcp2_frame_decoder *frd, return ngtcp2_pkt_decode_path_response_frame(&dest->path_response, payload, payloadlen); case NGTCP2_FRAME_CRYPTO: - dest->stream.data = &frd->buf.stream_data; + dest->stream.data = &frd->buf.data; return ngtcp2_pkt_decode_crypto_frame(&dest->stream, payload, payloadlen); case NGTCP2_FRAME_NEW_TOKEN: return ngtcp2_pkt_decode_new_token_frame(&dest->new_token, payload, @@ -556,11 +556,12 @@ ngtcp2_ssize ngtcp2_frame_decoder_decode(ngtcp2_frame_decoder *frd, payload, payloadlen); case NGTCP2_FRAME_DATAGRAM: case NGTCP2_FRAME_DATAGRAM_LEN: + dest->datagram.data = &frd->buf.data; return ngtcp2_pkt_decode_datagram_frame(&dest->datagram, payload, payloadlen); default: if ((type & ~(NGTCP2_FRAME_STREAM - 1)) == NGTCP2_FRAME_STREAM) { - dest->stream.data = &frd->buf.stream_data; + dest->stream.data = &frd->buf.data; return ngtcp2_pkt_decode_stream_frame(&dest->stream, payload, payloadlen); } @@ -1490,16 +1491,13 @@ ngtcp2_ssize ngtcp2_pkt_decode_datagram_frame(ngtcp2_datagram *dest, dest->type = type; - if (datalen == 0) { - dest->datacnt = 0; - dest->data = NULL; - } else { + if (datalen) { + dest->data[0].len = datalen; + dest->data[0].base = (uint8_t *)p; dest->datacnt = 1; - dest->data = dest->rdata; - dest->rdata[0].len = datalen; - - dest->rdata[0].base = (uint8_t *)p; p += datalen; + } else { + dest->datacnt = 0; } assert((size_t)(p - payload) == len); diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.h b/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.h index 78fcd6dcc2b54d..abf13693b916e7 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.h +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_pkt.h @@ -340,13 +340,9 @@ typedef struct ngtcp2_datagram { uint64_t dgram_id; /* datacnt is the number of elements that data contains. */ size_t datacnt; - /* data is a pointer to ngtcp2_vec array that stores data. */ + /* data is a pointer to ngtcp2_vec array that stores data. If + datacnt == 0, this field may be NULL.*/ ngtcp2_vec *data; - /* rdata is conveniently embedded to ngtcp2_datagram, so that data - field can just point to the address of this field to store a - single vector which is the case when DATAGRAM is received from a - remote endpoint. */ - ngtcp2_vec rdata[1]; } ngtcp2_datagram; typedef union ngtcp2_frame { @@ -457,7 +453,7 @@ ngtcp2_ssize ngtcp2_pkt_encode_hd_short(uint8_t *out, size_t outlen, */ typedef struct ngtcp2_frame_decoder { union { - ngtcp2_vec stream_data; + ngtcp2_vec data; ngtcp2_ack_range ack_ranges[NGTCP2_MAX_ACK_RANGES]; } buf; } ngtcp2_frame_decoder; diff --git a/deps/ngtcp2/ngtcp2/lib/ngtcp2_qlog.c b/deps/ngtcp2/ngtcp2/lib/ngtcp2_qlog.c index 7c1142d14d3c6a..c10e2ff6700451 100644 --- a/deps/ngtcp2/ngtcp2/lib/ngtcp2_qlog.c +++ b/deps/ngtcp2/ngtcp2/lib/ngtcp2_qlog.c @@ -223,28 +223,21 @@ static const ngtcp2_vec vec_pkt_type_stateless_reset = static const ngtcp2_vec vec_pkt_type_unknown = ngtcp2_make_vec_lit("unknown"); static const ngtcp2_vec *qlog_pkt_type(const ngtcp2_pkt_hd *hd) { - if (hd->flags & NGTCP2_PKT_FLAG_LONG_FORM) { - switch (hd->type) { - case NGTCP2_PKT_INITIAL: - return &vec_pkt_type_initial; - case NGTCP2_PKT_HANDSHAKE: - return &vec_pkt_type_handshake; - case NGTCP2_PKT_0RTT: - return &vec_pkt_type_0rtt; - case NGTCP2_PKT_RETRY: - return &vec_pkt_type_retry; - default: - return &vec_pkt_type_unknown; - } - } - switch (hd->type) { + case NGTCP2_PKT_INITIAL: + return &vec_pkt_type_initial; + case NGTCP2_PKT_0RTT: + return &vec_pkt_type_0rtt; + case NGTCP2_PKT_HANDSHAKE: + return &vec_pkt_type_handshake; + case NGTCP2_PKT_RETRY: + return &vec_pkt_type_retry; + case NGTCP2_PKT_1RTT: + return &vec_pkt_type_1rtt; case NGTCP2_PKT_VERSION_NEGOTIATION: return &vec_pkt_type_version_negotiation; case NGTCP2_PKT_STATELESS_RESET: return &vec_pkt_type_stateless_reset; - case NGTCP2_PKT_1RTT: - return &vec_pkt_type_1rtt; default: return &vec_pkt_type_unknown; } diff --git a/deps/simdjson/simdjson.cpp b/deps/simdjson/simdjson.cpp index 0374b89f39c2d3..8f3305987681e6 100644 --- a/deps/simdjson/simdjson.cpp +++ b/deps/simdjson/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2026-02-20 16:16:37 -0500. version 4.3.1 Do not edit! */ +/* auto-generated on 2026-03-25 17:25:37 -0400. version 4.5.0 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -146,7 +146,6 @@ #define SIMDJSON_CONSTEVAL 0 #endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #endif // !defined(SIMDJSON_CONSTEVAL) - #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -219,6 +218,25 @@ using std::size_t; #define SIMDJSON_IS_LASX 1 // We can always run both #elif defined(__loongarch_sx) #define SIMDJSON_IS_LSX 1 + +// Adjust for runtime dispatching support. +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__NVCOMPILER) +#if __GNUC__ > 15 || (__GNUC__ == 15 && __GNUC_MINOR__ >= 0) + // We are ok, we will support runtime dispatch for LASX. +#else + // We disable runtime dispatch for LASX, which means that we will not be able to use LASX + // even if it is supported by the hardware. + // Loongson users should update to GCC 15 or better. + #define SIMDJSON_IMPLEMENTATION_LASX 0 +#endif +#else + // We are not using GCC, so we assume that we can support runtime dispatch for LASX. + // https://godbolt.org/z/jcMnrjYhs + #define SIMDJSON_IMPLEMENTATION_LASX 0 +#endif + + + #endif #elif defined(__PPC64__) || defined(_M_PPC64) #define SIMDJSON_IS_PPC64 1 @@ -7290,6 +7308,12 @@ class dom_parser_implementation { */ size_t _max_depth{0}; +public: + /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ + bool _number_as_string{false}; + +protected: + // Declaring these so that subclasses can use them to implement their constructors. simdjson_inline dom_parser_implementation() noexcept; simdjson_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; @@ -7865,7 +7889,8 @@ enum class tape_type { DOUBLE = 'd', TRUE_VALUE = 't', FALSE_VALUE = 'f', - NULL_VALUE = 'n' + NULL_VALUE = 'n', + BIGINT = 'Z' // Big integer stored as string in string buffer }; // enum class tape_type } // namespace internal @@ -14160,6 +14185,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -14243,6 +14271,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace arm64 @@ -15180,7 +15220,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -20521,6 +20577,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -20604,6 +20663,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace haswell @@ -21541,7 +21612,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -26877,6 +26964,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -26960,6 +27050,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace icelake @@ -27897,7 +27999,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -33504,6 +33622,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -33587,6 +33708,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace ppc64 @@ -34524,7 +34657,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -40693,6 +40842,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -40776,6 +40928,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace westmere @@ -41713,7 +41877,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -46913,6 +47093,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -46996,6 +47179,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace lasx @@ -47933,7 +48128,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -53037,6 +53248,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -53120,6 +53334,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace lsx @@ -54057,7 +54283,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -59580,6 +59822,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -59663,6 +59908,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace rvv_vls @@ -60600,7 +60857,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { @@ -60938,6 +61211,19 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #endif// _MSC_VER } +simdjson_inline int trailing_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long trailing_zero = 0; + // Search the mask data from least significant bit (LSB) + // to most significant bit (MSB) for a set bit (1). + if (_BitScanForward64(&trailing_zero, input_num)) + return (int)trailing_zero; + else return 64; +#else + return __builtin_ctzll(input_num); +#endif// _MSC_VER +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -63169,6 +63455,19 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #endif// _MSC_VER } +simdjson_inline int trailing_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long trailing_zero = 0; + // Search the mask data from least significant bit (LSB) + // to most significant bit (MSB) for a set bit (1). + if (_BitScanForward64(&trailing_zero, input_num)) + return (int)trailing_zero; + else return 64; +#else + return __builtin_ctzll(input_num); +#endif// _MSC_VER +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -64145,6 +64444,9 @@ struct tape_writer { /** Write a double value to tape. */ simdjson_inline void append_double(double value) noexcept; + /** Write a big integer (as string) to tape. src points to first digit, len is byte count. */ + simdjson_inline void append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept; + /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ @@ -64228,6 +64530,18 @@ simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, intern tape_loc = val | ((uint64_t(char(t))) << 56); } +simdjson_inline void tape_writer::append_bigint(const uint8_t *src, size_t len, uint8_t *&string_buf) noexcept { + // Write to string buffer: [4-byte LE length][digits][null] + uint32_t str_len = uint32_t(len); + memcpy(string_buf, &str_len, sizeof(uint32_t)); + memcpy(string_buf + sizeof(uint32_t), src, len); + string_buf[sizeof(uint32_t) + len] = 0; + // Tape entry: offset into string buffer + // The caller must set the offset relative to doc.string_buf base + append(0, internal::tape_type::BIGINT); // placeholder offset, caller patches + string_buf += sizeof(uint32_t) + len + 1; +} + } // namespace stage2 } // unnamed namespace } // namespace fallback @@ -64411,7 +64725,23 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string( simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - return numberparsing::parse_number(value, tape); + error_code err = numberparsing::parse_number(value, tape); + if (simdjson_unlikely(err == BIGINT_ERROR && + iter.dom_parser._number_as_string)) { + // Write big integer to string buffer using the same format as strings. + // Scan digits the same way parse_number does (skip optional '-', then digits). + const uint8_t *p = value; + if (*p == '-') p++; + while (numberparsing::is_digit(*p)) p++; + size_t len = size_t(p - value); + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); + uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); + memcpy(dst, value, len); + dst += len; + on_end_string(dst); + return SUCCESS; + } + return err; } simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { diff --git a/deps/simdjson/simdjson.h b/deps/simdjson/simdjson.h index cf2658af259b79..3a413a7d1e046e 100644 --- a/deps/simdjson/simdjson.h +++ b/deps/simdjson/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2026-02-20 16:16:37 -0500. version 4.3.1 Do not edit! */ +/* auto-generated on 2026-03-25 17:25:37 -0400. version 4.5.0 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -166,7 +166,6 @@ #define SIMDJSON_CONSTEVAL 0 #endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #endif // !defined(SIMDJSON_CONSTEVAL) - #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -239,6 +238,25 @@ using std::size_t; #define SIMDJSON_IS_LASX 1 // We can always run both #elif defined(__loongarch_sx) #define SIMDJSON_IS_LSX 1 + +// Adjust for runtime dispatching support. +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) && !defined(__NVCOMPILER) +#if __GNUC__ > 15 || (__GNUC__ == 15 && __GNUC_MINOR__ >= 0) + // We are ok, we will support runtime dispatch for LASX. +#else + // We disable runtime dispatch for LASX, which means that we will not be able to use LASX + // even if it is supported by the hardware. + // Loongson users should update to GCC 15 or better. + #define SIMDJSON_IMPLEMENTATION_LASX 0 +#endif +#else + // We are not using GCC, so we assume that we can support runtime dispatch for LASX. + // https://godbolt.org/z/jcMnrjYhs + #define SIMDJSON_IMPLEMENTATION_LASX 0 +#endif + + + #endif #elif defined(__PPC64__) || defined(_M_PPC64) #define SIMDJSON_IS_PPC64 1 @@ -2520,7 +2538,7 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.3.1" +#define SIMDJSON_VERSION "4.5.0" namespace simdjson { enum { @@ -2531,11 +2549,11 @@ enum { /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 3, + SIMDJSON_VERSION_MINOR = 5, /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 1 + SIMDJSON_VERSION_REVISION = 0 }; } // namespace simdjson @@ -3774,6 +3792,12 @@ class dom_parser_implementation { */ size_t _max_depth{0}; +public: + /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ + bool _number_as_string{false}; + +protected: + // Declaring these so that subclasses can use them to implement their constructors. simdjson_inline dom_parser_implementation() noexcept; simdjson_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; @@ -4326,6 +4350,62 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } #endif + +#ifndef _WIN32 +/** + * A class representing a memory-mapped file with padding. + * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + */ +class padded_memory_map { +public: + /** + * Create a new padded memory map for the given file. + * After creating the memory map, you can call view() to get a padded_string_view of the file content. + * The memory map will be automatically released when the padded_memory_map instance is destroyed. + * Note that the file content is not copied, so this is efficient for large files. However, + * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, + * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * You can check if the memory map is valid by calling is_valid() before using view(). + * + * @param filename the path to the file to memory-map. + */ + simdjson_inline padded_memory_map(const char *filename) noexcept; + /** + * Destroy the padded memory map and release any resources. + */ + simdjson_inline ~padded_memory_map() noexcept; + + // lifetime of the view is tied to the memory map, so we can return a view + // directly + /** + * Get a view of the memory-mapped file. It always succeeds, but the view may be empty + * if the memory map is invalid (e.g., due to file not found, permission denied, etc.). + * You can check if the memory map is valid by calling is_valid() before using the view. + * + * Lifetime of the view is tied to the memory map, so the view should not be used after the + * padded_memory_map instance is destroyed. + * + * @return a padded_string_view representing the memory-mapped file, or an empty view if the memory map is invalid. + */ + simdjson_inline simdjson::padded_string_view view() const noexcept simdjson_lifetime_bound; + /** + * Check if the memory map is valid. + * + * @return true if the memory map is valid, false otherwise. + */ + simdjson_inline bool is_valid() const noexcept; + +private: + padded_memory_map() = delete; + padded_memory_map(const padded_memory_map &) = delete; + padded_memory_map &operator=(const padded_memory_map &) = delete; + const char *data{nullptr}; + size_t size{0}; +}; +#endif // _WIN32 + + + } // namespace simdjson // This is deliberately outside of simdjson so that people get it without having to use the namespace @@ -4404,7 +4484,7 @@ namespace simdjson { */ class padded_string_view : public std::string_view { private: - size_t _capacity; + size_t _capacity{0}; public: /** Create an empty padded_string_view. */ @@ -4608,6 +4688,14 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept { #include #include +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif + namespace simdjson { namespace internal { @@ -4964,6 +5052,57 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept { return true; } + +#ifndef _WIN32 +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + + int fd = open(filename, O_RDONLY); + if (fd == -1) { + return; // file not found or cannot be opened, data will be nullptr + } + struct stat st; + if (fstat(fd, &st) == -1) { + close(fd); + return; // failed to get file size, data will be nullptr + } + size = static_cast(st.st_size); + size_t total_size = size + simdjson::SIMDJSON_PADDING; + void *anon_map = + mmap(NULL, total_size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon_map == MAP_FAILED) { + close(fd); + return; // failed to create anonymous mapping, data will be nullptr + } + void *file_map = + mmap(anon_map, size, PROT_READ, MAP_SHARED | MAP_FIXED, fd, 0); + if (file_map == MAP_FAILED) { + munmap(anon_map, total_size); + close(fd); + return; // failed to mmap file, data will be nullptr + } + data = static_cast(file_map); + close(fd); // no longer needed after mapping +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data != nullptr) { + munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); + } +} + + +simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { + if(!is_valid()) { + return simdjson::padded_string_view(); // return an empty view if mapping failed + } + return simdjson::padded_string_view(data, size, size + simdjson::SIMDJSON_PADDING); +} + +simdjson_inline bool padded_memory_map::is_valid() const noexcept { + return data != nullptr; +} +#endif // _WIN32 + } // namespace simdjson inline simdjson::padded_string operator ""_padded(const char *str, size_t len) { @@ -4974,6 +5113,7 @@ inline simdjson::padded_string operator ""_padded(const char8_t *str, size_t len return simdjson::padded_string(reinterpret_cast(str), len); } #endif + #endif // SIMDJSON_PADDED_STRING_INL_H /* end file simdjson/padded_string-inl.h */ /* skipped duplicate #include "simdjson/padded_string_view.h" */ @@ -6030,6 +6170,13 @@ class parser { inline bool dump_raw_tape(std::ostream &os) const noexcept; + /** + * When enabled, big integers (exceeding uint64 range) are stored as strings + * in the tape instead of returning BIGINT_ERROR. Default: false. + */ + inline void number_as_string(bool enabled) noexcept { _number_as_string = enabled; } + inline bool number_as_string() const noexcept { return _number_as_string; } + private: /** * The maximum document length this parser will automatically support. @@ -6038,6 +6185,9 @@ class parser { */ size_t _max_capacity; + /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ + bool _number_as_string{false}; + /** * The loaded buffer (reused each time load() is called) */ @@ -6431,7 +6581,8 @@ enum class element_type { DOUBLE = 'd', ///< double: Any number with a "." or "e" that fits in double. STRING = '"', ///< std::string_view BOOL = 't', ///< bool - NULL_VALUE = 'n' ///< null + NULL_VALUE = 'n', ///< null + BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string }; /** @@ -6530,6 +6681,14 @@ class element { */ inline simdjson_result get_bool() const noexcept; + /** + * Read this element as a big integer (raw digit string). + * + * @returns A string_view of the raw digits, or: + * INCORRECT_TYPE if the JSON element is not a big integer. + */ + inline simdjson_result get_bigint() const noexcept; + /** * Whether this element is a json array. * @@ -6585,6 +6744,11 @@ class element { */ inline bool is_null() const noexcept; + /** + * Whether this element is a big integer (number exceeding 64-bit range). + */ + inline bool is_bigint() const noexcept; + /** * Tell whether the value can be cast to provided type (T). * @@ -6943,6 +7107,7 @@ struct simdjson_result : public internal::simdjson_result_base get_uint64() const noexcept; simdjson_inline simdjson_result get_double() const noexcept; simdjson_inline simdjson_result get_bool() const noexcept; + simdjson_inline simdjson_result get_bigint() const noexcept; simdjson_inline bool is_array() const noexcept; simdjson_inline bool is_object() const noexcept; @@ -6953,6 +7118,7 @@ struct simdjson_result : public internal::simdjson_result_base operator[](std::string_view key) const noexcept; simdjson_inline simdjson_result operator[](const char *key) const noexcept; @@ -7943,7 +8109,8 @@ enum class tape_type { DOUBLE = 'd', TRUE_VALUE = 't', FALSE_VALUE = 'f', - NULL_VALUE = 'n' + NULL_VALUE = 'n', + BIGINT = 'Z' // Big integer stored as string in string buffer }; // enum class tape_type } // namespace internal @@ -8817,6 +8984,10 @@ simdjson_inline simdjson_result simdjson_result::get_bool() if (error()) { return error(); } return first.get_bool(); } +simdjson_inline simdjson_result simdjson_result::get_bigint() const noexcept { + if (error()) { return error(); } + return first.get_bigint(); +} simdjson_inline bool simdjson_result::is_array() const noexcept { return !error() && first.is_array(); @@ -8846,6 +9017,9 @@ simdjson_inline bool simdjson_result::is_bool() const noexcept { simdjson_inline bool simdjson_result::is_null() const noexcept { return !error() && first.is_null(); } +simdjson_inline bool simdjson_result::is_bigint() const noexcept { + return !error() && first.is_bigint(); +} simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { if (error()) { return error(); } @@ -8954,6 +9128,15 @@ inline simdjson_result element::get_bool() const noexcept { } return INCORRECT_TYPE; } +inline simdjson_result element::get_bigint() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + switch (tape.tape_ref_type()) { + case internal::tape_type::BIGINT: + return tape.get_string_view(); + default: + return INCORRECT_TYPE; + } +} inline simdjson_result element::get_c_str() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { @@ -9096,6 +9279,10 @@ inline bool element::is_null() const noexcept { return tape.is_null_on_tape(); } +inline bool element::is_bigint() const noexcept { + return tape.tape_ref_type() == internal::tape_type::BIGINT; +} + #if SIMDJSON_EXCEPTIONS inline element::operator bool() const noexcept(false) { return get(); } @@ -9228,6 +9415,8 @@ inline std::ostream& operator<<(std::ostream& out, element_type type) { return out << "bool"; case element_type::NULL_VALUE: return out << "null"; + case element_type::BIGINT: + return out << "bigint"; default: return out << "unexpected content!!!"; // abort() usage is forbidden in the library } @@ -9395,6 +9584,7 @@ inline simdjson_result parser::parse_into_document(document& provided_d buf += 3; len -= 3; } + implementation->_number_as_string = _number_as_string; _error = implementation->parse(buf, len, provided_doc); if (_error) { return _error; } @@ -10073,6 +10263,15 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { case 'r': // we start and end with the root node // should we be hitting the root node? return false; + case 'Z': // we have a big integer + os << "bigint "; + std::memcpy(&string_length, string_buf.get() + payload, sizeof(uint32_t)); + os << std::string_view( + reinterpret_cast(string_buf.get() + payload + sizeof(uint32_t)), + string_length + ); + os << '\n'; + break; default: return false; } @@ -10558,6 +10757,12 @@ inline void string_builder::append(simdjson::dom::element value) { case tape_type::STRING: format.string(iter.get_string_view()); break; + case tape_type::BIGINT: { + // Big integer stored as string — output raw digits (no quotes) + auto sv = iter.get_string_view(); + format.chars(sv.data(), sv.data() + sv.size()); + break; + } case tape_type::INT64: format.number(iter.next_tape_value()); iter.json_index++; // numbers take up 2 spots, so we need to increment @@ -16498,6 +16703,19 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #endif// _MSC_VER } +simdjson_inline int trailing_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long trailing_zero = 0; + // Search the mask data from least significant bit (LSB) + // to most significant bit (MSB) for a set bit (1). + if (_BitScanForward64(&trailing_zero, input_num)) + return (int)trailing_zero; + else return 64; +#else + return __builtin_ctzll(input_num); +#endif// _MSC_VER +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -39592,6 +39810,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -39604,6 +39838,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace arm64 { @@ -39704,6 +39954,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -39747,28 +40026,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -39806,13 +40069,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -40506,6 +40884,19 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #endif// _MSC_VER } +simdjson_inline int trailing_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long trailing_zero = 0; + // Search the mask data from least significant bit (LSB) + // to most significant bit (MSB) for a set bit (1). + if (_BitScanForward64(&trailing_zero, input_num)) + return (int)trailing_zero; + else return 64; +#else + return __builtin_ctzll(input_num); +#endif// _MSC_VER +} + } // unnamed namespace } // namespace fallback } // namespace simdjson @@ -41503,6 +41894,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -41515,6 +41922,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace fallback { @@ -41615,6 +42038,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -41658,28 +42110,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -41717,13 +42153,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -43914,6 +44465,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -43926,6 +44493,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace haswell { @@ -44026,6 +44609,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -44069,28 +44681,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -44128,13 +44724,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -46325,6 +47036,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -46337,6 +47064,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace icelake { @@ -46437,6 +47180,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -46480,28 +47252,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -46539,13 +47295,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -48851,6 +49722,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -48863,6 +49750,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace ppc64 { @@ -48963,6 +49866,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -49006,28 +49938,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -49065,13 +49981,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -51694,6 +52725,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -51706,6 +52753,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace westmere { @@ -51806,6 +52869,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -51849,28 +52941,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -51908,13 +52984,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -54011,6 +55202,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -54023,6 +55230,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace lsx { @@ -54123,6 +55346,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -54166,28 +55418,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -54225,13 +55461,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -56351,6 +57702,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -56363,6 +57730,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace lasx { @@ -56463,6 +57846,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -56506,28 +57918,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -56565,13 +57961,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -58695,6 +60206,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 #endif #endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif #if SIMDJSON_EXPERIMENTAL_HAS_NEON #include #ifdef _MSC_VER @@ -58707,6 +60234,22 @@ simdjson_warn_unused simdjson_result extract_fractured_json( #include #endif #endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + namespace simdjson { namespace rvv_vls { @@ -58807,6 +60350,35 @@ simdjson_inline bool fast_needs_escaping(std::string_view view) { } return _mm_movemask_epi8(running) != 0; } +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline bool fast_needs_escaping(std::string_view view) { + if (view.size() < 16) { + return simple_needs_escaping(view); + } + size_t i = 0; + __vector unsigned char running = vec_splats((unsigned char)0); + __vector unsigned char v34 = vec_splats((unsigned char)34); + __vector unsigned char v92 = vec_splats((unsigned char)92); + __vector unsigned char v32 = vec_splats((unsigned char)32); + + for (; i + 15 < view.size(); i += 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(view.data() + i)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + if (i < view.size()) { + __vector unsigned char word = vec_vsx_ld( + 0, reinterpret_cast(view.data() + view.length() - 16)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v34)); + running = vec_or(running, (__vector unsigned char)vec_cmpeq(word, v92)); + running = vec_or(running, + (__vector unsigned char)vec_cmplt(word, v32)); + } + return !vec_all_eq(running, vec_splats((unsigned char)0)); +} #else simdjson_inline bool fast_needs_escaping(std::string_view view) { return simple_needs_escaping(view); @@ -58850,28 +60422,12 @@ find_next_json_quotable_character(const std::string_view view, needs_escape = vorrq_u8(needs_escape, vceqq_u8(word, v92)); needs_escape = vorrq_u8(needs_escape, vcltq_u8(word, v32)); - if (vmaxvq_u32(vreinterpretq_u32_u8(needs_escape)) != 0) { - // Found quotable character - extract exact byte position using ctz - uint64x2_t as64 = vreinterpretq_u64_u8(needs_escape); - uint64_t lo = vgetq_lane_u64(as64, 0); - uint64_t hi = vgetq_lane_u64(as64, 1); + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(needs_escape), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + if(mask != 0) { size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - if (lo != 0) { - _BitScanForward64(&trailing_zero, lo); - return offset + trailing_zero / 8; - } else { - _BitScanForward64(&trailing_zero, hi); - return offset + 8 + trailing_zero / 8; - } -#else - if (lo != 0) { - return offset + __builtin_ctzll(lo) / 8; - } else { - return offset + 8 + __builtin_ctzll(hi) / 8; - } -#endif + auto trailing_zero = trailing_zeroes(mask); + return offset + (trailing_zero >> 2); } ptr += 16; remaining -= 16; @@ -58909,13 +60465,128 @@ find_next_json_quotable_character(const std::string_view view, if (mask != 0) { // Found quotable character - use trailing zero count to find position size_t offset = ptr - reinterpret_cast(view.data()); -#ifdef _MSC_VER - unsigned long trailing_zero = 0; - _BitScanForward(&trailing_zero, mask); - return offset + trailing_zero; + return offset + trailing_zeroes(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); #else - return offset + __builtin_ctz(mask); + unsigned int mask = static_cast(result[0]); #endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); } ptr += 16; remaining -= 16; @@ -73462,6 +75133,19 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { #endif// _MSC_VER } +simdjson_inline int trailing_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long trailing_zero = 0; + // Search the mask data from least significant bit (LSB) + // to most significant bit (MSB) for a set bit (1). + if (_BitScanForward64(&trailing_zero, input_num)) + return (int)trailing_zero; + else return 64; +#else + return __builtin_ctzll(input_num); +#endif// _MSC_VER +} + } // unnamed namespace } // namespace fallback } // namespace simdjson diff --git a/deps/uv/docs/src/tcp.rst b/deps/uv/docs/src/tcp.rst index 94a3b2741e48f4..f9b203c41997d9 100644 --- a/deps/uv/docs/src/tcp.rst +++ b/deps/uv/docs/src/tcp.rst @@ -91,34 +91,6 @@ API .. versionchanged:: 1.49.0 If `delay` is less than 1 then ``UV_EINVAL``` is returned. -.. c:function:: int uv_tcp_keepalive_ex(uv_tcp_t* handle, int on, unsigned int idle, unsigned int intvl, unsigned int cnt) - - Enable / disable TCP keep-alive with all socket options: `TCP_KEEPIDLE`, `TCP_KEEPINTVL` and `TCP_KEEPCNT`. - `idle` is the value for `TCP_KEEPIDLE`, `intvl` is the value for `TCP_KEEPINTVL`, - `cnt` is the value for `TCP_KEEPCNT`, ignored when `on` is zero. - - With TCP keep-alive enabled, `idle` is the time (in seconds) the connection needs to remain idle before - TCP starts sending keep-alive probes. `intvl` is the time (in seconds) between individual keep-alive probes. - TCP will drop the connection after sending `cnt` probes without getting any replies from the peer, then the - handle is destroyed with a ``UV_ETIMEDOUT`` error passed to the corresponding callback. - - If one of `idle`, `intvl`, or `cnt` is less than 1, ``UV_EINVAL`` is returned. - - .. versionchanged:: 1.52.0 added support of setting `TCP_KEEPINTVL` and `TCP_KEEPCNT` socket options. - - .. note:: - Ensure that the socket options are supported by the underlying operating system. - Currently supported platforms: - - AIX - - DragonFlyBSD - - FreeBSD - - illumos - - Linux - - macOS - - NetBSD - - Solaris - - Windows - .. c:function:: int uv_tcp_simultaneous_accepts(uv_tcp_t* handle, int enable) Enable / disable simultaneous asynchronous accept requests that are diff --git a/deps/uv/include/uv.h b/deps/uv/include/uv.h index da06345557bb6c..bdd3d792cb667d 100644 --- a/deps/uv/include/uv.h +++ b/deps/uv/include/uv.h @@ -602,11 +602,6 @@ UV_EXTERN int uv_tcp_nodelay(uv_tcp_t* handle, int enable); UV_EXTERN int uv_tcp_keepalive(uv_tcp_t* handle, int enable, unsigned int delay); -UV_EXTERN int uv_tcp_keepalive_ex(uv_tcp_t* handle, - int on, - unsigned int idle, - unsigned int intvl, - unsigned int cnt); UV_EXTERN int uv_tcp_simultaneous_accepts(uv_tcp_t* handle, int enable); enum uv_tcp_flags { diff --git a/deps/uv/src/unix/internal.h b/deps/uv/src/unix/internal.h index 80fe41f8ef6236..08a490b0a0b56c 100644 --- a/deps/uv/src/unix/internal.h +++ b/deps/uv/src/unix/internal.h @@ -359,11 +359,7 @@ int uv__slurp(const char* filename, char* buf, size_t len); /* tcp */ int uv__tcp_listen(uv_tcp_t* tcp, int backlog, uv_connection_cb cb); int uv__tcp_nodelay(int fd, int on); -int uv__tcp_keepalive(int fd, - int on, - unsigned int idle, - unsigned int intvl, - unsigned int cnt); +int uv__tcp_keepalive(int fd, int on, unsigned int delay); /* tty */ void uv__tty_close(uv_tty_t* handle); diff --git a/deps/uv/src/unix/stream.c b/deps/uv/src/unix/stream.c index 08a18314b1ec4e..d9e564016295fc 100644 --- a/deps/uv/src/unix/stream.c +++ b/deps/uv/src/unix/stream.c @@ -416,7 +416,7 @@ int uv__stream_open(uv_stream_t* stream, int fd, int flags) { /* TODO Use delay the user passed in. */ if ((stream->flags & UV_HANDLE_TCP_KEEPALIVE) && - uv__tcp_keepalive(fd, 1, 60, 1, 10)) { + uv__tcp_keepalive(fd, 1, 60)) { return UV__ERR(errno); } } diff --git a/deps/uv/src/unix/tcp.c b/deps/uv/src/unix/tcp.c index aa81c16d6a95fc..95c82d7f721843 100644 --- a/deps/uv/src/unix/tcp.c +++ b/deps/uv/src/unix/tcp.c @@ -465,18 +465,22 @@ int uv__tcp_nodelay(int fd, int on) { #else #define UV_KEEPALIVE_FACTOR(x) #endif -int uv__tcp_keepalive(int fd, - int on, - unsigned int idle, - unsigned int intvl, - unsigned int cnt) { +int uv__tcp_keepalive(int fd, int on, unsigned int delay) { + int idle; + int intvl; + int cnt; + + (void) &idle; + (void) &intvl; + (void) &cnt; + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on))) return UV__ERR(errno); if (!on) return 0; - if (idle < 1 || intvl < 1 || cnt < 1) + if (delay < 1) return UV_EINVAL; #ifdef __sun @@ -502,16 +506,13 @@ int uv__tcp_keepalive(int fd, * The TCP connection will be aborted after certain amount of probes, which is set by TCP_KEEPCNT, without receiving response. */ - /* Kernel expects at least 10 seconds for TCP_KEEPIDLE and TCP_KEEPINTVL. */ + idle = delay; + /* Kernel expects at least 10 seconds. */ if (idle < 10) idle = 10; - if (intvl < 10) - intvl = 10; - /* Kernel expects at most 10 days for TCP_KEEPIDLE and TCP_KEEPINTVL. */ + /* Kernel expects at most 10 days. */ if (idle > 10*24*60*60) idle = 10*24*60*60; - if (intvl > 10*24*60*60) - intvl = 10*24*60*60; UV_KEEPALIVE_FACTOR(idle); @@ -521,10 +522,12 @@ int uv__tcp_keepalive(int fd, if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle))) return UV__ERR(errno); + intvl = 10; /* required at least 10 seconds */ UV_KEEPALIVE_FACTOR(intvl); if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl))) return UV__ERR(errno); + cnt = 1; /* 1 retry, ensure (TCP_KEEPINTVL * TCP_KEEPCNT) is 10 seconds */ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt))) return UV__ERR(errno); #else @@ -536,7 +539,7 @@ int uv__tcp_keepalive(int fd, /* Note that the consequent probes will not be sent at equal intervals on Solaris, * but will be sent using the exponential backoff algorithm. */ - unsigned int time_to_abort = intvl * cnt; + int time_to_abort = 10; /* 10 seconds */ UV_KEEPALIVE_FACTOR(time_to_abort); if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE_ABORT_THRESHOLD, &time_to_abort, sizeof(time_to_abort))) return UV__ERR(errno); @@ -544,6 +547,7 @@ int uv__tcp_keepalive(int fd, #else /* !defined(__sun) */ + idle = delay; UV_KEEPALIVE_FACTOR(idle); #ifdef TCP_KEEPIDLE if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle))) @@ -555,12 +559,14 @@ int uv__tcp_keepalive(int fd, #endif #ifdef TCP_KEEPINTVL + intvl = 1; /* 1 second; same as default on Win32 */ UV_KEEPALIVE_FACTOR(intvl); if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl))) return UV__ERR(errno); #endif #ifdef TCP_KEEPCNT + cnt = 10; /* 10 retries; same as hardcoded on Win32 */ if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt))) return UV__ERR(errno); #endif @@ -588,20 +594,11 @@ int uv_tcp_nodelay(uv_tcp_t* handle, int on) { } -int uv_tcp_keepalive(uv_tcp_t* handle, int on, unsigned int idle) { - return uv_tcp_keepalive_ex(handle, on, idle, 1, 10); -} - - -int uv_tcp_keepalive_ex(uv_tcp_t* handle, - int on, - unsigned int idle, - unsigned int intvl, - unsigned int cnt) { +int uv_tcp_keepalive(uv_tcp_t* handle, int on, unsigned int delay) { int err; if (uv__stream_fd(handle) != -1) { - err = uv__tcp_keepalive(uv__stream_fd(handle), on, idle, intvl, cnt); + err =uv__tcp_keepalive(uv__stream_fd(handle), on, delay); if (err) return err; } @@ -611,7 +608,7 @@ int uv_tcp_keepalive_ex(uv_tcp_t* handle, else handle->flags &= ~UV_HANDLE_TCP_KEEPALIVE; - /* TODO Store idle if uv__stream_fd(handle) == -1 but don't want to enlarge + /* TODO Store delay if uv__stream_fd(handle) == -1 but don't want to enlarge * uv_tcp_t with an int that's almost never used... */ diff --git a/deps/uv/src/win/tcp.c b/deps/uv/src/win/tcp.c index 66e005ff021f63..c452c12e8f06f1 100644 --- a/deps/uv/src/win/tcp.c +++ b/deps/uv/src/win/tcp.c @@ -49,98 +49,28 @@ static int uv__tcp_nodelay(uv_tcp_t* handle, SOCKET socket, int enable) { } -/* - * Check if Windows version is 10.0.16299 (Windows 10, version 1709) or later. - */ -static int uv__windows10_version1709(void) { - OSVERSIONINFOW os_info; - if (!pRtlGetVersion) - return 0; - pRtlGetVersion(&os_info); - if (os_info.dwMajorVersion < 10) - return 0; - if (os_info.dwMajorVersion > 10) - return 1; - if (os_info.dwMinorVersion > 0) - return 1; - return os_info.dwBuildNumber >= 16299; -} - - -static int uv__tcp_keepalive(uv_tcp_t* handle, - SOCKET socket, - int on, - unsigned int idle, - unsigned int intvl, - unsigned int cnt) { +static int uv__tcp_keepalive(uv_tcp_t* handle, SOCKET socket, int enable, unsigned int delay) { if (setsockopt(socket, SOL_SOCKET, SO_KEEPALIVE, - (const char*)&on, - sizeof on) == -1) { + (const char*)&enable, + sizeof enable) == -1) { return WSAGetLastError(); } - if (!on) + if (!enable) return 0; - if (idle < 1 || intvl < 1 || cnt < 1) + if (delay < 1) return UV_EINVAL; - /* Windows 10, version 1709 (build 10.0.16299) and later require second units - * for TCP keepalive options. */ - if (uv__windows10_version1709()) { - if (setsockopt(socket, - IPPROTO_TCP, - TCP_KEEPIDLE, - (const char*)&idle, - sizeof idle) == -1) { - return WSAGetLastError(); - } - - if (setsockopt(socket, - IPPROTO_TCP, - TCP_KEEPINTVL, - (const char*)&intvl, - sizeof intvl) == -1) { - return WSAGetLastError(); - } - - if (setsockopt(socket, - IPPROTO_TCP, - TCP_KEEPCNT, - (const char*)&cnt, - sizeof cnt) == -1) { - return WSAGetLastError(); - } - - return 0; - } - - /* For those versions prior to Windows 10 version 1709, - * we fall back to SIO_KEEPALIVE_VALS that expects millisecond units. - * The SIO_KEEPALIVE_VALS IOCTL is supported on Windows 2000 - * and later versions of the operating system. */ - struct tcp_keepalive keepalive; - keepalive.onoff = on; - keepalive.keepalivetime = idle * 1000; - keepalive.keepaliveinterval = intvl * 1000; - /* On Windows Vista and later, the number of keep-alive probes - * (data retransmissions) is set to 10 and cannot be changed. - * On Windows Server 2003, Windows XP, and Windows 2000, the default setting - * for number of keep-alive probes is 5 and cannot be changed programmatically. - */ - DWORD dummy; - if (WSAIoctl(socket, - SIO_KEEPALIVE_VALS, - (LPVOID) &keepalive, - sizeof keepalive, - NULL, - 0, - &dummy, - NULL, - NULL) == -1) + if (setsockopt(socket, + IPPROTO_TCP, + TCP_KEEPALIVE, + (const char*)&delay, + sizeof delay) == -1) { return WSAGetLastError(); + } return 0; } @@ -202,7 +132,7 @@ static int uv__tcp_set_socket(uv_loop_t* loop, /* TODO: Use stored delay. */ if (handle->flags & UV_HANDLE_TCP_KEEPALIVE) { - err = uv__tcp_keepalive(handle, socket, 1, 60, 1, 10); + err = uv__tcp_keepalive(handle, socket, 1, 60); if (err) return err; } @@ -819,6 +749,20 @@ static int uv__is_loopback(const struct sockaddr_storage* storage) { return 0; } +// Check if Windows version is 10.0.16299 or later +static int uv__is_fast_loopback_fail_supported(void) { + OSVERSIONINFOW os_info; + if (!pRtlGetVersion) + return 0; + pRtlGetVersion(&os_info); + if (os_info.dwMajorVersion < 10) + return 0; + if (os_info.dwMajorVersion > 10) + return 1; + if (os_info.dwMinorVersion > 0) + return 1; + return os_info.dwBuildNumber >= 16299; +} static int uv__tcp_try_connect(uv_connect_t* req, uv_tcp_t* handle, @@ -865,7 +809,7 @@ static int uv__tcp_try_connect(uv_connect_t* req, * is not reachable, instead of waiting for 2s. We do not care if this fails. * This only works on Windows version 10.0.16299 and later. */ - if (uv__windows10_version1709() && uv__is_loopback(&converted)) { + if (uv__is_fast_loopback_fail_supported() && uv__is_loopback(&converted)) { memset(&retransmit_ioctl, 0, sizeof(retransmit_ioctl)); retransmit_ioctl.Rtt = TCP_INITIAL_RTO_NO_SYN_RETRANSMISSIONS; retransmit_ioctl.MaxSynRetransmissions = TCP_INITIAL_RTO_NO_SYN_RETRANSMISSIONS; @@ -1391,30 +1335,22 @@ int uv_tcp_nodelay(uv_tcp_t* handle, int enable) { } -int uv_tcp_keepalive(uv_tcp_t* handle, int on, unsigned int idle) { - return uv_tcp_keepalive_ex(handle, on, idle, 1, 10); -} - -int uv_tcp_keepalive_ex(uv_tcp_t* handle, - int on, - unsigned int idle, - unsigned int intvl, - unsigned int cnt) { +int uv_tcp_keepalive(uv_tcp_t* handle, int enable, unsigned int delay) { int err; if (handle->socket != INVALID_SOCKET) { - err = uv__tcp_keepalive(handle, handle->socket, on, idle, intvl, cnt); + err = uv__tcp_keepalive(handle, handle->socket, enable, delay); if (err) return uv_translate_sys_error(err); } - if (on) { + if (enable) { handle->flags |= UV_HANDLE_TCP_KEEPALIVE; } else { handle->flags &= ~UV_HANDLE_TCP_KEEPALIVE; } - /* TODO: Store idle if handle->socket isn't created yet. */ + /* TODO: Store delay if handle->socket isn't created yet. */ return 0; } diff --git a/deps/uv/src/win/winsock.h b/deps/uv/src/win/winsock.h index e14b8b515a9935..bb3808a35c27e6 100644 --- a/deps/uv/src/win/winsock.h +++ b/deps/uv/src/win/winsock.h @@ -38,6 +38,10 @@ # define SO_UPDATE_CONNECT_CONTEXT 0x7010 #endif +#ifndef TCP_KEEPALIVE +# define TCP_KEEPALIVE 3 +#endif + #ifndef IPV6_V6ONLY # define IPV6_V6ONLY 27 #endif @@ -58,30 +62,6 @@ # define MCAST_LEAVE_SOURCE_GROUP 46 #endif -#ifndef SIO_KEEPALIVE_VALS -#define SIO_KEEPALIVE_VALS _WSAIOW(IOC_VENDOR,4) -struct tcp_keepalive { - u_long onoff; - u_long keepalivetime; - u_long keepaliveinterval; -}; -#endif - -/* - * TCP keepalive definitions on MinGW are located in . - */ -#ifndef TCP_KEEPIDLE -#define TCP_KEEPIDLE 0x03 /* start keepalives after this period */ -#endif - -#ifndef TCP_KEEPINTVL -#define TCP_KEEPINTVL 0x11 /* interval between keepalives */ -#endif - -#ifndef TCP_KEEPCNT -#define TCP_KEEPCNT 0x10 /* number of keepalives before death */ -#endif - /* * TDI defines that are only in the DDK. * We only need receive flags so far. diff --git a/deps/uv/test/test-tcp-flags.c b/deps/uv/test/test-tcp-flags.c index 04a238b2c4cd49..16218a27f0a3a8 100644 --- a/deps/uv/test/test-tcp-flags.c +++ b/deps/uv/test/test-tcp-flags.c @@ -49,21 +49,6 @@ TEST_IMPL(tcp_flags) { r = uv_tcp_keepalive(&handle, 1, 0); ASSERT_EQ(r, UV_EINVAL); - r = uv_tcp_keepalive_ex(&handle, 1, 60, 60, 60); - ASSERT_OK(r); - - r = uv_tcp_keepalive_ex(&handle, 0, 0, 0, 0); - ASSERT_OK(r); - - r = uv_tcp_keepalive_ex(&handle, 1, 0, 10, 10); - ASSERT_EQ(r, UV_EINVAL); - - r = uv_tcp_keepalive_ex(&handle, 1, 10, 0, 10); - ASSERT_EQ(r, UV_EINVAL); - - r = uv_tcp_keepalive_ex(&handle, 1, 10, 10, 0); - ASSERT_EQ(r, UV_EINVAL); - uv_close((uv_handle_t*)&handle, NULL); r = uv_run(loop, UV_RUN_DEFAULT); diff --git a/doc/api/sqlite.md b/doc/api/sqlite.md index a5c5fd6cec2b0d..35bbebea16e158 100644 --- a/doc/api/sqlite.md +++ b/doc/api/sqlite.md @@ -82,6 +82,29 @@ console.log(query.all()); // Prints: [ { key: 1, value: 'hello' }, { key: 2, value: 'world' } ] ``` +## Type conversion between JavaScript and SQLite + +When Node.js writes to or reads from SQLite, it is necessary to convert between +JavaScript data types and SQLite's [data types][]. Because JavaScript supports +more data types than SQLite, only a subset of JavaScript types are supported. +Attempting to write an unsupported data type to SQLite will result in an +exception. + +| Storage class | JavaScript to SQLite | SQLite to JavaScript | +| ------------- | -------------------------- | ------------------------------------- | +| `NULL` | {null} | {null} | +| `INTEGER` | {number} or {bigint} | {number} or {bigint} _(configurable)_ | +| `REAL` | {number} | {number} | +| `TEXT` | {string} | {string} | +| `BLOB` | {TypedArray} or {DataView} | {Uint8Array} | + +APIs that read values from SQLite have a configuration option that determines +whether `INTEGER` values are converted to `number` or `bigint` in JavaScript, +such as the `readBigInts` option for statements and the `useBigIntArguments` +option for user-defined functions. If Node.js reads an `INTEGER` value from +SQLite that is outside the JavaScript [safe integer][] range, and the option to +read BigInts is not enabled, then an `ERR_OUT_OF_RANGE` error will be thrown. + ## Class: `DatabaseSync`