From e39a2e62f4265feb94512bf54d734abbb4fc71ea Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 27 Apr 2023 10:48:49 -0500 Subject: [PATCH] Add option to locally enforce payload size limit (#515) Add a configuration option to enforce an item size limit on the client side. This avoids sending large items over the wire and getting rejected by the server which can cause delays. The default is 0 for no limit. The same error code RES_E2BIG is used for the client side limit as for the server side limit. --- memcached.ini | 6 +++++ php_memcached.c | 45 ++++++++++++++++++++++++++++++++++++++ php_memcached_private.h | 5 +++-- tests/cas_e2big.phpt | 32 +++++++++++++++++++++++++++ tests/options.phpt | 32 +++++++++++++++++++++++++++ tests/set_large.phpt | 4 +++- tests/set_large_e2big.phpt | 27 +++++++++++++++++++++++ tests/setoptions.phpt | 3 +++ 8 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 tests/cas_e2big.phpt create mode 100644 tests/set_large_e2big.phpt diff --git a/memcached.ini b/memcached.ini index c54e1fd..5decf39 100644 --- a/memcached.ini +++ b/memcached.ini @@ -133,6 +133,12 @@ ; the default is 0 ;memcached.store_retry_count = 0 +; The maximum payload size in bytes that can be written. +; Writing a payload larger than the limit will result in RES_E2BIG error. +; Specifying 0 means no limit is enforced, though the server may still reject with RES_E2BIG. +; Default is 0. +;memcached.item_size_limit = 1000000 + ; Sets the default for consistent hashing for new connections. ; (To configure consistent hashing for session connections, ; use memcached.sess_consistent_hash instead) diff --git a/php_memcached.c b/php_memcached.c index e7d5736..9c5ba8a 100644 --- a/php_memcached.c +++ b/php_memcached.c @@ -82,6 +82,7 @@ static int php_memc_list_entry(void) { #define MEMC_OPT_STORE_RETRY_COUNT -1005 #define MEMC_OPT_USER_FLAGS -1006 #define MEMC_OPT_COMPRESSION_LEVEL -1007 +#define MEMC_OPT_ITEM_SIZE_LIMIT -1008 /**************************************** Custom result codes @@ -162,6 +163,7 @@ typedef struct { zend_long store_retry_count; zend_long set_udf_flags; + zend_long item_size_limit; #ifdef HAVE_MEMCACHED_SASL zend_bool has_sasl_data; @@ -423,6 +425,7 @@ PHP_INI_BEGIN() MEMC_INI_ENTRY("compression_threshold", "2000", OnUpdateLong, compression_threshold) MEMC_INI_ENTRY("serializer", SERIALIZER_DEFAULT_NAME, OnUpdateSerializer, serializer_name) MEMC_INI_ENTRY("store_retry_count", "0", OnUpdateLong, store_retry_count) + MEMC_INI_ENTRY("item_size_limit", "0", OnUpdateLongGEZero, item_size_limit) MEMC_INI_BOOL ("default_consistent_hash", "0", OnUpdateBool, default_behavior.consistent_hash_enabled) MEMC_INI_BOOL ("default_binary_protocol", "0", OnUpdateBool, default_behavior.binary_protocol_enabled) @@ -1127,6 +1130,21 @@ zend_string *s_zval_to_payload(php_memc_object_t *intern, zval *value, uint32_t return payload; } +static +zend_bool s_is_payload_too_big(php_memc_object_t *intern, zend_string *payload) +{ + php_memc_user_data_t *memc_user_data = memcached_get_user_data(intern->memc); + + /* An item size limit of 0 implies no limit enforced */ + if (memc_user_data->item_size_limit == 0) { + return 0; + } + if (ZSTR_LEN(payload) > memc_user_data->item_size_limit) { + return 1; + } + return 0; +} + static zend_bool s_should_retry_write (php_memc_object_t *intern, memcached_return status) { @@ -1153,6 +1171,12 @@ zend_bool s_memc_write_zval (php_memc_object_t *intern, php_memc_write_op op, ze s_memc_set_status(intern, MEMC_RES_PAYLOAD_FAILURE, 0); return 0; } + + if (s_is_payload_too_big(intern, payload)) { + s_memc_set_status(intern, MEMCACHED_E2BIG, 0); + zend_string_release(payload); + return 0; + } } #define memc_write_using_fn(fn_name) payload ? fn_name(intern->memc, ZSTR_VAL(key), ZSTR_LEN(key), ZSTR_VAL(payload), ZSTR_LEN(payload), expiration, flags) : MEMC_RES_PAYLOAD_FAILURE; @@ -1305,6 +1329,7 @@ static PHP_METHOD(Memcached, __construct) memc_user_data->encoding_enabled = 0; memc_user_data->store_retry_count = MEMC_G(store_retry_count); memc_user_data->set_udf_flags = -1; + memc_user_data->item_size_limit = MEMC_G(item_size_limit); memc_user_data->is_persistent = is_persistent; memcached_set_user_data(intern->memc, memc_user_data); @@ -2145,6 +2170,12 @@ static void php_memc_cas_impl(INTERNAL_FUNCTION_PARAMETERS, zend_bool by_key) RETURN_FALSE; } + if (s_is_payload_too_big(intern, payload)) { + intern->rescode = MEMCACHED_E2BIG; + zend_string_release(payload); + RETURN_FALSE; + } + if (by_key) { status = memcached_cas_by_key(intern->memc, ZSTR_VAL(server_key), ZSTR_LEN(server_key), ZSTR_VAL(key), ZSTR_LEN(key), ZSTR_VAL(payload), ZSTR_LEN(payload), expiration, flags, cas); } else { @@ -2970,6 +3001,9 @@ static PHP_METHOD(Memcached, getOption) case MEMC_OPT_COMPRESSION: RETURN_BOOL(memc_user_data->compression_enabled); + case MEMC_OPT_ITEM_SIZE_LIMIT: + RETURN_LONG(memc_user_data->item_size_limit); + case MEMC_OPT_PREFIX_KEY: { memcached_return retval; @@ -3041,6 +3075,15 @@ int php_memc_set_option(php_memc_object_t *intern, long option, zval *value) } break; + case MEMC_OPT_ITEM_SIZE_LIMIT: + lval = zval_get_long(value); + if (lval < 0) { + php_error_docref(NULL, E_WARNING, "ITEM_SIZE_LIMIT must be >= 0"); + return 0; + } + memc_user_data->item_size_limit = lval; + break; + case MEMC_OPT_PREFIX_KEY: { zend_string *str; @@ -4013,6 +4056,7 @@ PHP_GINIT_FUNCTION(php_memcached) php_memcached_globals->memc.compression_factor = 1.30; php_memcached_globals->memc.compression_level = 3; php_memcached_globals->memc.store_retry_count = 2; + php_memcached_globals->memc.item_size_limit = 0; php_memcached_globals->memc.sasl_initialised = 0; php_memcached_globals->no_effect = 0; @@ -4063,6 +4107,7 @@ static void php_memc_register_constants(INIT_FUNC_ARGS) REGISTER_MEMC_CLASS_CONST_LONG(OPT_USER_FLAGS, MEMC_OPT_USER_FLAGS); REGISTER_MEMC_CLASS_CONST_LONG(OPT_STORE_RETRY_COUNT, MEMC_OPT_STORE_RETRY_COUNT); + REGISTER_MEMC_CLASS_CONST_LONG(OPT_ITEM_SIZE_LIMIT, MEMC_OPT_ITEM_SIZE_LIMIT); /* * Indicate whether igbinary serializer is available diff --git a/php_memcached_private.h b/php_memcached_private.h index aefaf4f..93c2057 100644 --- a/php_memcached_private.h +++ b/php_memcached_private.h @@ -186,8 +186,9 @@ ZEND_BEGIN_MODULE_GLOBALS(php_memcached) char *compression_name; zend_long compression_threshold; double compression_factor; - zend_long store_retry_count; - zend_long compression_level; + zend_long store_retry_count; + zend_long compression_level; + zend_long item_size_limit; /* Converted values*/ php_memc_serializer_type serializer_type; diff --git a/tests/cas_e2big.phpt b/tests/cas_e2big.phpt new file mode 100644 index 0000000..99c3562 --- /dev/null +++ b/tests/cas_e2big.phpt @@ -0,0 +1,32 @@ +--TEST-- +set data exceeding size limit +--SKIPIF-- + +--FILE-- + 100, +)); + +$m->delete('cas_e2big_test'); + +$m->set('cas_e2big_test', 'hello'); +$result = $m->get('cas_e2big_test', null, Memcached::GET_EXTENDED); +var_dump(is_array($result) && isset($result['cas']) && isset($result['value']) && $result['value'] == 'hello'); + +$value = str_repeat('a large payload', 1024 * 1024); + +var_dump($m->cas($result['cas'], 'cas_e2big_test', $value, 360)); +var_dump($m->getResultCode() == Memcached::RES_E2BIG); +var_dump($m->getResultMessage() == 'ITEM TOO BIG'); +var_dump($m->get('cas_e2big_test') == 'hello'); +var_dump($m->getResultCode() == Memcached::RES_SUCCESS); +?> +--EXPECT-- +bool(true) +bool(false) +bool(true) +bool(true) +bool(true) +bool(true) diff --git a/tests/options.phpt b/tests/options.phpt index ce89538..a096c8f 100644 --- a/tests/options.phpt +++ b/tests/options.phpt @@ -26,6 +26,26 @@ var_dump($m->getOption(Memcached::OPT_COMPRESSION_TYPE) == Memcached::COMPRESSIO var_dump($m->setOption(Memcached::OPT_COMPRESSION_TYPE, 0)); var_dump($m->getOption(Memcached::OPT_COMPRESSION_TYPE) == Memcached::COMPRESSION_FASTLZ); + +echo "item_size_limit setOption\n"; +var_dump($m->setOption(Memcached::OPT_ITEM_SIZE_LIMIT, 0)); +var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) === 0); +var_dump($m->setOption(Memcached::OPT_ITEM_SIZE_LIMIT, -1)); +var_dump($m->setOption(Memcached::OPT_ITEM_SIZE_LIMIT, 1000000)); +var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) == 1000000); + +echo "item_size_limit ini\n"; +ini_set('memcached.item_size_limit', '0'); +$m = new Memcached(); +var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) === 0); + +ini_set('memcached.item_size_limit', '1000000'); +$m = new Memcached(); +var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) == 1000000); + +ini_set('memcached.item_size_limit', null); +$m = new Memcached(); +var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) === 0); ?> --EXPECTF-- bool(true) @@ -41,3 +61,15 @@ bool(true) bool(true) bool(false) bool(true) +item_size_limit setOption +bool(true) +bool(true) + +Warning: Memcached::setOption(): ITEM_SIZE_LIMIT must be >= 0 in %s on line %d +bool(false) +bool(true) +bool(true) +item_size_limit ini +bool(true) +bool(true) +bool(true) diff --git a/tests/set_large.phpt b/tests/set_large.phpt index f3cb4cc..ba9fbca 100644 --- a/tests/set_large.phpt +++ b/tests/set_large.phpt @@ -5,7 +5,9 @@ set large data --FILE-- 0, +)); $key = 'foobarbazDEADC0DE'; $value = str_repeat("foo bar", 1024 * 1024); diff --git a/tests/set_large_e2big.phpt b/tests/set_large_e2big.phpt new file mode 100644 index 0000000..498231e --- /dev/null +++ b/tests/set_large_e2big.phpt @@ -0,0 +1,27 @@ +--TEST-- +set data exceeding size limit +--SKIPIF-- + +--FILE-- + 100, +)); + +$m->delete('set_large_e2big_test'); + +$value = str_repeat('a large payload', 1024 * 1024); + +var_dump($m->set('set_large_e2big_test', $value)); +var_dump($m->getResultCode() == Memcached::RES_E2BIG); +var_dump($m->getResultMessage() == 'ITEM TOO BIG'); +var_dump($m->get('set_large_e2big_test') === false); +var_dump($m->getResultCode() == Memcached::RES_NOTFOUND); +?> +--EXPECT-- +bool(false) +bool(true) +bool(true) +bool(true) +bool(true) diff --git a/tests/setoptions.phpt b/tests/setoptions.phpt index 4bf39d1..9cefd13 100644 --- a/tests/setoptions.phpt +++ b/tests/setoptions.phpt @@ -13,12 +13,14 @@ var_dump($m->setOptions(array( Memcached::OPT_COMPRESSION => 0, Memcached::OPT_LIBKETAMA_COMPATIBLE => 1, Memcached::OPT_CONNECT_TIMEOUT => 5000, + Memcached::OPT_ITEM_SIZE_LIMIT => 1000000, ))); var_dump($m->getOption(Memcached::OPT_PREFIX_KEY) == 'a_prefix'); var_dump($m->getOption(Memcached::OPT_SERIALIZER) == Memcached::SERIALIZER_PHP); var_dump($m->getOption(Memcached::OPT_COMPRESSION) == 0); var_dump($m->getOption(Memcached::OPT_LIBKETAMA_COMPATIBLE) == 1); +var_dump($m->getOption(Memcached::OPT_ITEM_SIZE_LIMIT) == 1000000); echo "test invalid options\n"; @@ -36,6 +38,7 @@ bool(true) bool(true) bool(true) bool(true) +bool(true) test invalid options Warning: Memcached::setOptions(): invalid configuration option in %s on line %d