diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 1f603497..3c21f1ce 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -11,7 +11,7 @@ packages: - py311-sphinx - py311-m2r - rsync - - tbb + - onetbb sources: - https://github.com/awesomized/libmemcached secrets: diff --git a/docs/source/libmemcached/index_basics.rst b/docs/source/libmemcached/index_basics.rst index 49f2f74d..a3db2175 100644 --- a/docs/source/libmemcached/index_basics.rst +++ b/docs/source/libmemcached/index_basics.rst @@ -6,6 +6,7 @@ Basics memcached_create memcached_get + memcached_gat memcached_set memcached_delete memcached_quit diff --git a/docs/source/libmemcached/memcached_gat.rst b/docs/source/libmemcached/memcached_gat.rst new file mode 100644 index 00000000..97c5c028 --- /dev/null +++ b/docs/source/libmemcached/memcached_gat.rst @@ -0,0 +1,87 @@ +Get and update expiration atomically +===================================== + +SYNOPSIS +-------- + +#include + Compile and link with -lmemcached + +.. function:: char *memcached_gat (memcached_st *ptr, const char *key, size_t key_length, time_t expiration, size_t *value_length, uint32_t *flags, memcached_return_t *error) + + :param ptr: pointer to initialized `memcached_st` struct + :param key: the key to fetch and touch + :param key_length: the length of `key` without any terminating zero + :param expiration: new expiration as a unix timestamp or as relative expiration time in seconds + :param value_length: pointer filled with the length of the returned value, or ``NULL`` + :param flags: pointer filled with the flags stored with the value, or ``NULL`` + :param error: pointer filled with the return status, or ``NULL`` + :returns: pointer to the retrieved value, or ``NULL`` on error or cache miss + +.. function:: char *memcached_gat_by_key (memcached_st *ptr, const char *group_key, size_t group_key_length, const char *key, size_t key_length, time_t expiration, size_t *value_length, uint32_t *flags, memcached_return_t *error) + + :param ptr: pointer to initialized `memcached_st` struct + :param group_key: the key namespace used to select the server + :param group_key_length: the length of `group_key` without any terminating zero + :param key: the key to fetch and touch + :param key_length: the length of `key` without any terminating zero + :param expiration: new expiration as a unix timestamp or as relative expiration time in seconds + :param value_length: pointer filled with the length of the returned value, or ``NULL`` + :param flags: pointer filled with the flags stored with the value, or ``NULL`` + :param error: pointer filled with the return status, or ``NULL`` + :returns: pointer to the retrieved value, or ``NULL`` on error or cache miss + +DESCRIPTION +----------- + +:func:`memcached_gat` (Get And Touch) atomically fetches the value for a key +and updates its expiration time in a single round trip to the server. It +combines the semantics of :func:`memcached_get` and :func:`memcached_touch`: +the current value is returned to the caller while the TTL is refreshed. + +:func:`memcached_gat_by_key` works identically but accepts a `group_key` +that controls which server the key is located on, enabling key partitioning. + +Both functions support the text protocol (``gat``/``gats``) and the binary +protocol (``GATK`` opcode). When `MEMCACHED_BEHAVIOR_SUPPORT_CAS` is +enabled the text protocol uses ``gats``, and the response includes a CAS +token accessible via :func:`memcached_result_cas`. + +The returned value must be released by the caller using :manpage:`free(3)`. + +These functions are not supported when `MEMCACHED_BEHAVIOR_USE_UDP` is set; +that behavior returns `MEMCACHED_NOT_SUPPORTED`. + +RETURN VALUE +------------ + +On success, a pointer to the retrieved value is returned and `*error` is set +to `MEMCACHED_SUCCESS`. The caller must free this pointer with +:manpage:`free(3)`. + +``NULL`` is returned when: + +* The key does not exist â `*error` is set to `MEMCACHED_NOTFOUND`. +* A network or protocol error occurred â `*error` describes the failure. + +Use :func:`memcached_strerror` to convert a :type:`memcached_return_t` value +to a human-readable string. + +SEE ALSO +-------- + +.. only:: man + + :manpage:`memcached(1)` + :manpage:`libmemcached(3)` + :manpage:`memcached_get(3)` + :manpage:`memcached_touch(3)` + :manpage:`memcached_strerror(3)` + +.. only:: html + + * :manpage:`memcached(1)` + * :doc:`../libmemcached` + * :doc:`memcached_get` + * :doc:`memcached_touch` + * :doc:`memcached_strerror` diff --git a/include/libmemcached-1.0/CMakeLists.txt b/include/libmemcached-1.0/CMakeLists.txt index c9b4f7c1..59bd2fdb 100644 --- a/include/libmemcached-1.0/CMakeLists.txt +++ b/include/libmemcached-1.0/CMakeLists.txt @@ -25,6 +25,7 @@ install_public_headers( fetch.h flush_buffers.h flush.h + gat.h get.h hash.h limits.h diff --git a/include/libmemcached-1.0/gat.h b/include/libmemcached-1.0/gat.h new file mode 100644 index 00000000..43f32b80 --- /dev/null +++ b/include/libmemcached-1.0/gat.h @@ -0,0 +1,33 @@ +/* + +--------------------------------------------------------------------+ + | libmemcached-awesome - C/C++ Client Library for memcached | + +--------------------------------------------------------------------+ + | Redistribution and use in source and binary forms, with or without | + | modification, are permitted under the terms of the BSD license. | + | You should have received a copy of the license in a bundled file | + | named LICENSE; in case you did not receive a copy you can review | + | the terms online at: https://opensource.org/licenses/BSD-3-Clause | + +--------------------------------------------------------------------+ + | Copyright (c) 2006-2014 Brian Aker https://datadifferential.com/ | + | Copyright (c) 2020-2021 Michael Wallner https://awesome.co/ | + +--------------------------------------------------------------------+ +*/ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +LIBMEMCACHED_API +char *memcached_gat(memcached_st *ptr, const char *key, size_t key_length, time_t expiration, + size_t *value_length, uint32_t *flags, memcached_return_t *error); + +LIBMEMCACHED_API +char *memcached_gat_by_key(memcached_st *ptr, const char *group_key, size_t group_key_length, + const char *key, size_t key_length, time_t expiration, + size_t *value_length, uint32_t *flags, memcached_return_t *error); + +#ifdef __cplusplus +} +#endif diff --git a/include/libmemcached-1.0/memcached.h b/include/libmemcached-1.0/memcached.h index e744de29..d21a37d5 100644 --- a/include/libmemcached-1.0/memcached.h +++ b/include/libmemcached-1.0/memcached.h @@ -64,6 +64,7 @@ #include "libmemcached-1.0/fetch.h" #include "libmemcached-1.0/flush.h" #include "libmemcached-1.0/flush_buffers.h" +#include "libmemcached-1.0/gat.h" #include "libmemcached-1.0/get.h" #include "libmemcached-1.0/hash.h" #include "libmemcached-1.0/options.h" diff --git a/src/libmemcached/CMakeLists.txt b/src/libmemcached/CMakeLists.txt index 05116ae8..ebf5a963 100644 --- a/src/libmemcached/CMakeLists.txt +++ b/src/libmemcached/CMakeLists.txt @@ -47,6 +47,7 @@ set(libmemcached_sources flag.cc flush.cc flush_buffers.cc + gat.cc get.cc hash.cc hosts.cc diff --git a/src/libmemcached/gat.cc b/src/libmemcached/gat.cc new file mode 100644 index 00000000..caf31843 --- /dev/null +++ b/src/libmemcached/gat.cc @@ -0,0 +1,162 @@ +/* + +--------------------------------------------------------------------+ + | libmemcached-awesome - C/C++ Client Library for memcached | + +--------------------------------------------------------------------+ + | Redistribution and use in source and binary forms, with or without | + | modification, are permitted under the terms of the BSD license. | + | You should have received a copy of the license in a bundled file | + | named LICENSE; in case you did not receive a copy you can review | + | the terms online at: https://opensource.org/licenses/BSD-3-Clause | + +--------------------------------------------------------------------+ + | Copyright (c) 2006-2014 Brian Aker https://datadifferential.com/ | + | Copyright (c) 2020-2021 Michael Wallner https://awesome.co/ | + +--------------------------------------------------------------------+ +*/ + +#include "libmemcached/common.h" + +static memcached_return_t ascii_gat(memcached_instance_st *instance, const char *key, + size_t key_length, time_t expiration) { + char expiration_buffer[MEMCACHED_MAXIMUM_INTEGER_DISPLAY_LENGTH + 1 + 1]; + int expiration_buffer_length = snprintf(expiration_buffer, sizeof(expiration_buffer), "%lld", + (long long) expiration); + if (size_t(expiration_buffer_length) >= sizeof(expiration_buffer) + 1 + or expiration_buffer_length < 0) + { + return memcached_set_error( + *instance, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT, + memcached_literal_param("snprintf(MEMCACHED_MAXIMUM_INTEGER_DISPLAY_LENGTH)")); + } + + /* Use "gats" when CAS support is requested so the VALUE response includes the CAS token */ + const char *gat_command; + uint8_t gat_command_length; + if (instance->root->flags.support_cas) { + gat_command = "gats "; + gat_command_length = 5; + } else { + gat_command = "gat "; + gat_command_length = 4; + } + + libmemcached_io_vector_st vector[] = { + {NULL, 0}, + {gat_command, gat_command_length}, + {expiration_buffer, size_t(expiration_buffer_length)}, + {memcached_literal_param(" ")}, + {memcached_array_string(instance->root->_namespace), + memcached_array_size(instance->root->_namespace)}, + {key, key_length}, + {memcached_literal_param("\r\n")}}; + + memcached_return_t rc; + if (memcached_failed(rc = memcached_vdo(instance, vector, 7, true))) { + return memcached_set_error(*instance, MEMCACHED_WRITE_FAILURE, MEMCACHED_AT); + } + + return rc; +} + +static memcached_return_t binary_gat(memcached_instance_st *instance, const char *key, + size_t key_length, time_t expiration) { + protocol_binary_request_gat request = {}; + + initialize_binary_request(instance, request.message.header); + + /* GATK returns the key in the response, matching how GETK is used for binary get */ + request.message.header.request.opcode = PROTOCOL_BINARY_CMD_GATK; + request.message.header.request.extlen = 4; + request.message.header.request.keylen = + htons((uint16_t)(key_length + memcached_array_size(instance->root->_namespace))); + request.message.header.request.datatype = PROTOCOL_BINARY_RAW_BYTES; + request.message.header.request.bodylen = + htonl((uint32_t)(key_length + memcached_array_size(instance->root->_namespace) + + request.message.header.request.extlen)); + request.message.body.expiration = htonl((uint32_t) expiration); + + libmemcached_io_vector_st vector[] = { + {NULL, 0}, + {request.bytes, sizeof(request.bytes)}, + {memcached_array_string(instance->root->_namespace), + memcached_array_size(instance->root->_namespace)}, + {key, key_length}}; + + memcached_return_t rc; + if (memcached_failed(rc = memcached_vdo(instance, vector, 4, true))) { + return memcached_set_error(*instance, MEMCACHED_WRITE_FAILURE, MEMCACHED_AT); + } + + return rc; +} + +char *memcached_gat(memcached_st *ptr, const char *key, size_t key_length, time_t expiration, + size_t *value_length, uint32_t *flags, memcached_return_t *error) { + return memcached_gat_by_key(ptr, NULL, 0, key, key_length, expiration, value_length, flags, + error); +} + +char *memcached_gat_by_key(memcached_st *shell, const char *group_key, size_t group_key_length, + const char *key, size_t key_length, time_t expiration, + size_t *value_length, uint32_t *flags, memcached_return_t *error) { + Memcached *ptr = memcached2Memcached(shell); + memcached_return_t unused; + if (error == NULL) { + error = &unused; + } + + memcached_return_t rc; + if (memcached_failed(rc = initialize_query(ptr, true))) { + *error = rc; + if (value_length) { + *value_length = 0; + } + return NULL; + } + + if (memcached_is_udp(ptr)) { + *error = memcached_set_error(*ptr, MEMCACHED_NOT_SUPPORTED, MEMCACHED_AT); + if (value_length) { + *value_length = 0; + } + return NULL; + } + + if (memcached_failed(rc = memcached_key_test(*ptr, (const char **) &key, &key_length, 1))) { + *error = memcached_set_error(*ptr, rc, MEMCACHED_AT); + if (value_length) { + *value_length = 0; + } + return NULL; + } + + uint32_t server_key; + if (group_key and group_key_length) { + server_key = memcached_generate_hash_with_redistribution(ptr, group_key, group_key_length); + } else { + server_key = memcached_generate_hash_with_redistribution(ptr, key, key_length); + } + memcached_instance_st *instance = memcached_instance_fetch(ptr, server_key); + + if (ptr->flags.binary_protocol) { + rc = binary_gat(instance, key, key_length, expiration); + } else { + rc = ascii_gat(instance, key, key_length, expiration); + } + + if (memcached_failed(rc)) { + *error = rc; + if (value_length) { + *value_length = 0; + } + return NULL; + } + + char *value = memcached_fetch(ptr, NULL, NULL, value_length, flags, error); + + /* Normalize END (no key found) to NOTFOUND, matching memcached_get behavior */ + if (*error == MEMCACHED_END) { + *error = MEMCACHED_NOTFOUND; + } + + return value; +} diff --git a/src/libmemcached/response.cc b/src/libmemcached/response.cc index d3945133..afa0c079 100644 --- a/src/libmemcached/response.cc +++ b/src/libmemcached/response.cc @@ -469,13 +469,17 @@ static memcached_return_t binary_read_one_response(memcached_instance_st *instan or header.response.status == PROTOCOL_BINARY_RESPONSE_AUTH_CONTINUE) { switch (header.response.opcode) { + case PROTOCOL_BINARY_CMD_GATKQ: + /* fall through */ case PROTOCOL_BINARY_CMD_GETKQ: /* - * We didn't increment the response counter for the GETKQ packet + * We didn't increment the response counter for the GETKQ/GATKQ packet * (only the final NOOP), so we need to increment the counter again. */ memcached_server_response_increment(instance); /* fall through */ + case PROTOCOL_BINARY_CMD_GATK: + /* fall through */ case PROTOCOL_BINARY_CMD_GETK: { uint16_t keylen = header.response.keylen; memcached_result_reset(result); diff --git a/test/tests/memcached/gat.cpp b/test/tests/memcached/gat.cpp new file mode 100644 index 00000000..2b5c4d77 --- /dev/null +++ b/test/tests/memcached/gat.cpp @@ -0,0 +1,106 @@ +#include "test/lib/common.hpp" +#include "test/lib/MemcachedCluster.hpp" + +TEST_CASE("memcached_gat") { + auto test = MemcachedCluster::mixed(); + auto memc = &test.memc; + memcached_return_t rc; + auto binary = GENERATE(0, 1); + + test.enableBinaryProto(binary); + + DYNAMIC_SECTION("gat missing key binary=" << binary) { + REQUIRE_FALSE(memcached_gat(memc, S("gat_missing"), 60, nullptr, nullptr, &rc)); + REQUIRE_RC(MEMCACHED_NOTFOUND, rc); + } + + DYNAMIC_SECTION("gat returns value binary=" << binary) { + REQUIRE_SUCCESS(memcached_set(memc, S("gat_key"), S("hello"), 60, 0)); + + size_t len; + uint32_t flags; + Malloced val(memcached_gat(memc, S("gat_key"), 120, &len, &flags, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val); + REQUIRE(string("hello") == string(*val, len)); + REQUIRE(flags == 0); + } + + DYNAMIC_SECTION("gat extends expiration binary=" << binary) { + /* Set with short TTL then extend via gat */ + REQUIRE_SUCCESS(memcached_set(memc, S("gat_extend"), S("value"), 2, 0)); + + Malloced val(memcached_gat(memc, S("gat_extend"), 60, nullptr, nullptr, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val); + + /* Wait past the original 2s TTL to prove gat actually extended it */ + this_thread::sleep_for(3s); + + Malloced val2(memcached_get(memc, S("gat_extend"), nullptr, nullptr, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val2); + } + + DYNAMIC_SECTION("gat expires key immediately binary=" << binary) { + REQUIRE_SUCCESS(memcached_set(memc, S("gat_expire"), S("soon"), 60, 0)); + + /* GAT with a past timestamp causes the key to expire immediately; the + value is still returned for this call (server touches then returns) */ + Malloced val(memcached_gat(memc, S("gat_expire"), time(nullptr) - 2, nullptr, nullptr, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val); + + /* Key is now expired */ + Malloced val2(memcached_get(memc, S("gat_expire"), nullptr, nullptr, &rc)); + REQUIRE_RC(MEMCACHED_NOTFOUND, rc); + REQUIRE_FALSE(*val2); + } + + DYNAMIC_SECTION("gat preserves flags binary=" << binary) { + REQUIRE_SUCCESS(memcached_set(memc, S("gat_flags"), S("flagged"), 60, 42)); + + uint32_t flags; + Malloced val(memcached_gat(memc, S("gat_flags"), 60, nullptr, &flags, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val); + REQUIRE(flags == 42); + } + + DYNAMIC_SECTION("gat_by_key missing key binary=" << binary) { + Malloced val(memcached_gat_by_key(memc, S("group"), S("gat_by_key_missing"), 60, + nullptr, nullptr, &rc)); + REQUIRE_FALSE(*val); + REQUIRE_RC(MEMCACHED_NOTFOUND, rc); + } + + DYNAMIC_SECTION("gat_by_key returns value binary=" << binary) { + REQUIRE_SUCCESS( + memcached_set_by_key(memc, S("group"), S("gat_by_key_val"), S("world"), 60, 0)); + + size_t len; + Malloced val(memcached_gat_by_key(memc, S("group"), S("gat_by_key_val"), 120, + &len, nullptr, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val); + REQUIRE(string("world") == string(*val, len)); + } + + DYNAMIC_SECTION("gat_by_key extends expiration binary=" << binary) { + REQUIRE_SUCCESS( + memcached_set_by_key(memc, S("group"), S("gat_by_key_ext"), S("extend"), 2, 0)); + + Malloced val(memcached_gat_by_key(memc, S("group"), S("gat_by_key_ext"), 60, + nullptr, nullptr, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val); + + /* Wait past the original 2s TTL to prove gat_by_key actually extended it */ + this_thread::sleep_for(3s); + + Malloced val2(memcached_get_by_key(memc, S("group"), S("gat_by_key_ext"), + nullptr, nullptr, &rc)); + REQUIRE_SUCCESS(rc); + REQUIRE(*val2); + } +}