php-win-ext/php-memcached
1
0
Fork 0
mirror of https://github.com/php-win-ext/php-memcached.git synced 2026-03-24 00:52:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

zstd support (#539)

Browse Source 
This adds zstd compression support.

The current two options, zlib and fastlz is basically a choice between performance and compression ratio.
You would choose zlib if you are memory-bound and fastlz if you are cpu-bound. With zstd, you get the
performance of fastlz with the compression of zlib. And often it wins on both. See this benchmark I ran
on json files of varying sizes: https://gist.github.com/rlerdorf/788f3d0144f9c5514d8fee9477cbe787

Taking just a 40k json blob, we see that zstd at compression level 3 reduces it to 8862 bytes. Our current
zlib 1 gets worse compression at 10091 bytes and takes longer both to compress and decompress.

      C Size  ratio%     C MB/s     D MB/s   SCORE      Name            File
        8037    19.9       0.58    2130.89       0.08   zstd 22         file-39.54k-json
        8204    20.3      31.85    2381.59       0.01   zstd 10         file-39.54k-json
        8371    20.7      47.52     547.12       0.01   zlib 9          file-39.54k-json
        8477    20.9      74.84     539.83       0.01   zlib 6          file-39.54k-json
        8862    21.9     449.86    2130.89       0.01   zstd 3          file-39.54k-json
        9171    22.7     554.62    2381.59       0.01   zstd 1          file-39.54k-json
       10091    24.9     153.94     481.99       0.01   zlib 1          file-39.54k-json
       10646    26.3      43.39    8097.40       0.01   lz4 16          file-39.54k-json
       10658    26.3      72.30    8097.40       0.01   lz4 10          file-39.54k-json
       13004    32.1    1396.10    6747.83       0.01   lz4 1           file-39.54k-json
       13321    32.9     440.08    1306.03       0.01   fastlz 2        file-39.54k-json
       14807    36.6     444.91    1156.77       0.01   fastlz 1        file-39.54k-json
       15517    38.3    1190.79    4048.70       0.02   zstd -10        file-39.54k-json

The fact that decompression a dramatically faster with zstd is a win for most common memcache uses
since they tend to be read-heavy. The PR also adds a `memcache.compression_level` INI switch which
currently only applies to zstd compression. It could probably be made to also apply to zlib and fastlz.
This commit is contained in:
Rasmus Lerdorf
2023-04-27 11:32:10 -04:00
committed by GitHub
parent 6926c53ac3
commit 5833590296
5 changed files with 125 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
config.m4
View File

@@ -27,6 +27,9 @@ PHP_ARG_ENABLE(memcached-protocol, whether to enable memcached protocol support,
PHP_ARG_WITH(system-fastlz, whether to use system FastLZ library,
[ --with-system-fastlz Use system FastLZ library], no, no)
PHP_ARG_WITH(zstd, whether to use system zstd library,
[ --with-zstd Use system zstd library], no, no)
if test -z "$PHP_ZLIB_DIR"; then
PHP_ARG_WITH(zlib-dir, for ZLIB,
[ --with-zlib-dir=DIR Set the path to ZLIB install prefix.], no)
@@ -345,6 +348,13 @@ if test "$PHP_MEMCACHED" != "no"; then
PHP_MEMCACHED_FILES="${PHP_MEMCACHED_FILES} fastlz/fastlz.c"
fi
if test "$PHP_ZSTD" != "no"; then
AC_CHECK_HEADERS([zstd.h], [ac_cv_have_zstd="yes"], [ac_cv_have_zstd="no"])
PHP_CHECK_LIBRARY(zstd, ZSTD_compress,
[PHP_ADD_LIBRARY(zstd, 1, MEMCACHED_SHARED_LIBADD)],
[AC_MSG_ERROR(zstd library not found)])
fi
if test "$PHP_MEMCACHED_SESSION" != "no"; then
PHP_MEMCACHED_FILES="${PHP_MEMCACHED_FILES} php_memcached_session.c"
fi

78
php_memcached.c
View File

@@ -37,6 +37,10 @@
#endif
#include <zlib.h>
#ifdef HAVE_ZSTD_H
#include <zstd.h>
#endif
#ifdef HAVE_JSON_API
# include "ext/json/php_json.h"
#endif
@@ -77,6 +81,7 @@ static int php_memc_list_entry(void) {
#define MEMC_OPT_COMPRESSION_TYPE -1004
#define MEMC_OPT_STORE_RETRY_COUNT -1005
#define MEMC_OPT_USER_FLAGS -1006
#define MEMC_OPT_COMPRESSION_LEVEL -1007
/****************************************
Custom result codes
@@ -107,6 +112,7 @@ static int php_memc_list_entry(void) {
#define MEMC_VAL_COMPRESSED (1<<0)
#define MEMC_VAL_COMPRESSION_ZLIB (1<<1)
#define MEMC_VAL_COMPRESSION_FASTLZ (1<<2)
#define MEMC_VAL_COMPRESSION_ZSTD (1<<3)
#define MEMC_VAL_GET_FLAGS(internal_flags) (((internal_flags) & MEMC_MASK_INTERNAL) >> 4)
#define MEMC_VAL_SET_FLAG(internal_flags, internal_flag) ((internal_flags) |= (((internal_flag) << 4) & MEMC_MASK_INTERNAL))
@@ -152,6 +158,7 @@ typedef struct {
zend_long serializer;
zend_long compression_type;
zend_long compression_level;
zend_long store_retry_count;
zend_long set_udf_flags;
@@ -278,6 +285,10 @@ static PHP_INI_MH(OnUpdateCompressionType)
MEMC_G(compression_type) = COMPRESSION_TYPE_FASTLZ;
} else if (!strcmp(ZSTR_VAL(new_value), "zlib")) {
MEMC_G(compression_type) = COMPRESSION_TYPE_ZLIB;
#ifdef HAVE_ZSTD_H
} else if (!strcmp(ZSTR_VAL(new_value), "zstd")) {
MEMC_G(compression_type) = COMPRESSION_TYPE_ZSTD;
#endif
} else {
return FAILURE;
}
@@ -408,6 +419,7 @@ PHP_INI_BEGIN()
MEMC_INI_ENTRY("compression_type", "fastlz", OnUpdateCompressionType, compression_name)
MEMC_INI_ENTRY("compression_factor", "1.3", OnUpdateReal, compression_factor)
MEMC_INI_ENTRY("compression_level", "3", OnUpdateLong, compression_level)
MEMC_INI_ENTRY("compression_threshold", "2000", OnUpdateLong, compression_threshold)
MEMC_INI_ENTRY("serializer", SERIALIZER_DEFAULT_NAME, OnUpdateSerializer, serializer_name)
MEMC_INI_ENTRY("store_retry_count", "0", OnUpdateLong, store_retry_count)
@@ -897,6 +909,19 @@ zend_bool s_compress_value (php_memc_compression_type compression_type, zend_str
}
break;
#ifdef HAVE_ZSTD_H
case COMPRESSION_TYPE_ZSTD:
{
compressed_size = ZSTD_compress((void *)buffer, buffer_size, ZSTR_VAL(payload), ZSTR_LEN(payload), MEMC_G(compression_level));
if (!ZSTD_isError(compressed_size)) {
compress_status = 1;
compression_type_flag = MEMC_VAL_COMPRESSION_ZSTD;
}
}
break;
#endif
case COMPRESSION_TYPE_ZLIB:
{
compressed_size = buffer_size;
@@ -2939,6 +2964,9 @@ static PHP_METHOD(Memcached, getOption)
case MEMC_OPT_COMPRESSION_TYPE:
RETURN_LONG(memc_user_data->compression_type);
case MEMC_OPT_COMPRESSION_LEVEL:
RETURN_LONG(memc_user_data->compression_level);
case MEMC_OPT_COMPRESSION:
RETURN_BOOL(memc_user_data->compression_enabled);
@@ -3001,6 +3029,9 @@ int php_memc_set_option(php_memc_object_t *intern, long option, zval *value)
case MEMC_OPT_COMPRESSION_TYPE:
lval = zval_get_long(value);
if (lval == COMPRESSION_TYPE_FASTLZ ||
#ifdef HAVE_ZSTD_H
lval == COMPRESSION_TYPE_ZSTD ||
#endif
lval == COMPRESSION_TYPE_ZLIB) {
memc_user_data->compression_type = lval;
} else {
@@ -3608,16 +3639,24 @@ zend_string *s_decompress_value (const char *payload, size_t payload_len, uint32
uint32_t stored_length;
unsigned long length;
zend_bool decompress_status = 0;
zend_bool is_fastlz = 0, is_zlib = 0;
zend_bool is_fastlz = 0, is_zlib = 0, is_zstd = 0;
if (payload_len < sizeof (uint32_t)) {
return NULL;
}
is_fastlz = MEMC_VAL_HAS_FLAG(flags, MEMC_VAL_COMPRESSION_FASTLZ);
is_zstd = MEMC_VAL_HAS_FLAG(flags, MEMC_VAL_COMPRESSION_ZSTD);
is_zlib = MEMC_VAL_HAS_FLAG(flags, MEMC_VAL_COMPRESSION_ZLIB);
if (!is_fastlz && !is_zlib) {
#ifndef HAVE_ZSTD_H
if (is_zstd) {
php_error_docref(NULL, E_WARNING, "could not decompress value: value was compressed with zstd but zstd support has not been compiled in");
return NULL;
}
#endif
if (!is_fastlz && !is_zlib && !is_zstd) {
php_error_docref(NULL, E_WARNING, "could not decompress value: unrecognised compression type");
return NULL;
}
@@ -3629,6 +3668,23 @@ zend_string *s_decompress_value (const char *payload, size_t payload_len, uint32
buffer = zend_string_alloc (stored_length, 0);
#ifdef HAVE_ZSTD_H
if (is_zstd) {
length = ZSTD_getFrameContentSize(payload, payload_len);
if (length == ZSTD_CONTENTSIZE_ERROR) {
php_error_docref(NULL, E_WARNING, "value was not compressed by zstd");
zend_string_release (buffer);
return NULL;
} else if (length == ZSTD_CONTENTSIZE_UNKNOWN) {
php_error_docref(NULL, E_WARNING, "zstd streaming decompression not supported");
zend_string_release (buffer);
return NULL;
}
decompress_status = !ZSTD_isError(ZSTD_decompress(&buffer->val, buffer->len, payload, payload_len));
}
else
#endif
if (is_fastlz) {
decompress_status = ((length = fastlz_decompress(payload, payload_len, &buffer->val, buffer->len)) > 0);
}
@@ -3955,6 +4011,7 @@ PHP_GINIT_FUNCTION(php_memcached)
php_memcached_globals->memc.compression_threshold = 2000;
php_memcached_globals->memc.compression_type = COMPRESSION_TYPE_FASTLZ;
php_memcached_globals->memc.compression_factor = 1.30;
php_memcached_globals->memc.compression_level = 3;
php_memcached_globals->memc.store_retry_count = 2;
php_memcached_globals->memc.sasl_initialised = 0;
@@ -4000,6 +4057,7 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
REGISTER_MEMC_CLASS_CONST_LONG(OPT_COMPRESSION, MEMC_OPT_COMPRESSION);
REGISTER_MEMC_CLASS_CONST_LONG(OPT_COMPRESSION_TYPE, MEMC_OPT_COMPRESSION_TYPE);
REGISTER_MEMC_CLASS_CONST_LONG(OPT_COMPRESSION_LEVEL, MEMC_OPT_COMPRESSION_LEVEL);
REGISTER_MEMC_CLASS_CONST_LONG(OPT_PREFIX_KEY, MEMC_OPT_PREFIX_KEY);
REGISTER_MEMC_CLASS_CONST_LONG(OPT_SERIALIZER, MEMC_OPT_SERIALIZER);
@@ -4015,6 +4073,15 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
REGISTER_MEMC_CLASS_CONST_BOOL(HAVE_IGBINARY, 0);
#endif
/*
* Indicate whether zstd compression is available
*/
#ifdef HAVE_ZSTD_H
REGISTER_MEMC_CLASS_CONST_BOOL(HAVE_ZSTD, 1);
#else
REGISTER_MEMC_CLASS_CONST_BOOL(HAVE_ZSTD, 0);
#endif
/*
* Indicate whether json serializer is available
*/
@@ -4186,6 +4253,7 @@ static void php_memc_register_constants(INIT_FUNC_ARGS)
*/
REGISTER_MEMC_CLASS_CONST_LONG(COMPRESSION_FASTLZ, COMPRESSION_TYPE_FASTLZ);
REGISTER_MEMC_CLASS_CONST_LONG(COMPRESSION_ZLIB, COMPRESSION_TYPE_ZLIB);
REGISTER_MEMC_CLASS_CONST_LONG(COMPRESSION_ZSTD, COMPRESSION_TYPE_ZSTD);
/*
* Flags.
@@ -4351,6 +4419,12 @@ PHP_MINFO_FUNCTION(memcached)
php_info_print_table_row(2, "msgpack support", "no");
#endif
#ifdef HAVE_ZSTD_H
php_info_print_table_row(2, "zstd support", "yes");
#else
php_info_print_table_row(2, "zstd support", "no");
#endif
php_info_print_table_end();
DISPLAY_INI_ENTRIES();

4
php_memcached_private.h
View File

@@ -98,7 +98,8 @@ typedef enum {
typedef enum {
COMPRESSION_TYPE_ZLIB = 1,
COMPRESSION_TYPE_FASTLZ = 2
COMPRESSION_TYPE_FASTLZ = 2,
COMPRESSION_TYPE_ZSTD = 3
} php_memc_compression_type;
typedef struct {
@@ -186,6 +187,7 @@ ZEND_BEGIN_MODULE_GLOBALS(php_memcached)
zend_long compression_threshold;
double compression_factor;
zend_long store_retry_count;
zend_long compression_level;
/* Converted values*/
php_memc_serializer_type serializer_type;

2
tests/compression_conditions.phpt
View File

@@ -21,6 +21,8 @@ function get_compression($name) {
return Memcached::COMPRESSION_ZLIB;
case 'fastlz':
return Memcached::COMPRESSION_FASTLZ;
case 'zstd':
return Memcached::COMPRESSION_ZSTD;
default:
echo "Strange compression type: $name\n";
return 0;

34
tests/compression_types.phpt
View File

@@ -15,6 +15,10 @@ function get_compression($name) {
return Memcached::COMPRESSION_ZLIB;
case 'fastlz':
return Memcached::COMPRESSION_FASTLZ;
case 'zstd':
if (Memcached::HAVE_ZSTD) {
return Memcached::COMPRESSION_ZSTD;
} else return 0;
default:
echo "Strange compression type: $name\n";
return 0;
@@ -54,6 +58,26 @@ fetch_with_compression($m, 'hello6', $data, '', 'fastlz');
fetch_with_compression($m, 'hello7', $data, 'zlib', '');
fetch_with_compression($m, 'hello8', $data, 'fastlz', '');
fetch_with_compression($m, 'hello9', $data, '', '');
if (Memcached::HAVE_ZSTD) {
fetch_with_compression($m, 'hello10', $data, 'zstd', 'zstd');
fetch_with_compression($m, 'hello11', $data, 'zstd', 'fastlz');
fetch_with_compression($m, 'hello12', $data, 'fastlz', 'zstd');
fetch_with_compression($m, 'hello13', $data, '', 'zstd');
fetch_with_compression($m, 'hello14', $data, 'zstd', '');
} else {
echo <<<EOB
set=[zstd] get=[zstd]
bool(true)
set=[zstd] get=[fastlz]
bool(true)
set=[fastlz] get=[zstd]
bool(true)
set=[] get=[zstd]
bool(true)
set=[zstd] get=[]
bool(true)
EOB;
}
?>
--EXPECT--
set=[zlib] get=[zlib]
@@ -74,3 +98,13 @@ set=[fastlz] get=[]
bool(true)
set=[] get=[]
bool(true)
set=[zstd] get=[zstd]
bool(true)
set=[zstd] get=[fastlz]
bool(true)
set=[fastlz] get=[zstd]
bool(true)
set=[] get=[zstd]
bool(true)
set=[zstd] get=[]
bool(true)