Issue #1048

This commit is adding support of data compression. LZF was choosen because it small and fast and Redis server uses it. Since [pecl package](https://pecl.php.net/package/lzf) doesn't provide lzf.h file after installing, LZF library was added as submodule. Another algorythms may be easely added by analogy with serializers. TODO: unit-tests for different data types.
2026-03-24 00:52:16 +01:00 · 2017-05-03 15:54:28 +03:00
parent b9ca16cc5b
commit 8cb2d5bd94
13 changed files with 151 additions and 27 deletions
--- a/.gitmodules
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "liblzf"]
+	path = liblzf
+	url = https://github.com/nemequ/liblzf.git
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,11 +30,11 @@ addons:
 before_install:
  - phpize
  - pecl install igbinary
-  - ./configure --enable-redis-igbinary CFLAGS=-Wall
+  - ./configure --enable-redis-igbinary --enable-redis-lzf CFLAGS=-Wall
 install: make install
 before_script:
  - gem install redis
-  - mkdir -p tests/nodes/ && echo >> tests/nodes/nodemap
+  - mkdir -p tests/nodes/ && echo > tests/nodes/nodemap
  - for PORT in $(seq 6379 6382); do redis-server --port $PORT --daemonize yes; done
  - for PORT in $(seq 7000 7011); do redis-server --port $PORT --cluster-enabled yes --cluster-config-file $PORT.conf --daemonize yes; echo 127.0.0.1:$PORT >> tests/nodes/nodemap; done
  - wget https://raw.githubusercontent.com/antirez/redis/unstable/src/redis-trib.rb
--- a/cluster_library.c
+++ b/cluster_library.c
@@ -456,7 +456,7 @@ void cluster_dist_add_val(redisCluster *c, clusterKeyVal *kv, zval *z_val
    int val_free;

    // Serialize our value
-    val_free = redis_serialize(c->flags, z_val, &val, &val_len TSRMLS_CC);
+    val_free = redis_pack(c->flags, z_val, &val, &val_len TSRMLS_CC);

    // Attach it to the provied keyval entry
    kv->val = val;
@@ -1495,12 +1495,12 @@ PHP_REDIS_API void cluster_bulk_resp(INTERNAL_FUNCTION_PARAMETERS, redisCluster
    }

    if (CLUSTER_IS_ATOMIC(c)) {
-        if (!redis_unserialize(c->flags, resp, c->reply_len, return_value TSRMLS_CC)) {
+        if (!redis_unpack(c->flags, resp, c->reply_len, return_value TSRMLS_CC)) {
            CLUSTER_RETURN_STRING(c, resp, c->reply_len);
        }
    } else {
        zval zv, *z = &zv;
-        if (redis_unserialize(c->flags, resp, c->reply_len, z TSRMLS_CC)) {
+        if (redis_unpack(c->flags, resp, c->reply_len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
            MAKE_STD_ZVAL(z);
            *z = zv;
@@ -2322,7 +2322,7 @@ int mbulk_resp_loop(RedisSock *redis_sock, zval *z_result,

        if (line != NULL) {
            zval zv, *z = &zv;
-            if (redis_unserialize(redis_sock, line, line_len, z TSRMLS_CC)) {
+            if (redis_unpack(redis_sock, line, line_len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
                MAKE_STD_ZVAL(z);
                *z = zv;
@@ -2367,7 +2367,7 @@ int mbulk_resp_loop_zipstr(RedisSock *redis_sock, zval *z_result,
        } else {
            /* Attempt serialization */
            zval zv, *z = &zv;
-            if (redis_unserialize(redis_sock, line, line_len, z TSRMLS_CC)) {
+            if (redis_unpack(redis_sock, line, line_len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
                MAKE_STD_ZVAL(z);
                *z = zv;
@@ -2406,7 +2406,7 @@ int mbulk_resp_loop_zipdbl(RedisSock *redis_sock, zval *z_result,
                key_len = line_len;
            } else {
                zval zv, *z = &zv;
-                if (redis_unserialize(redis_sock,key,key_len, z TSRMLS_CC)) {
+                if (redis_unpack(redis_sock,key,key_len, z TSRMLS_CC)) {
                    zend_string *zstr = zval_get_string(z);
                    add_assoc_double_ex(z_result, ZSTR_VAL(zstr), ZSTR_LEN(zstr), atof(line));
                    zend_string_release(zstr);
@@ -2440,7 +2440,7 @@ int mbulk_resp_loop_assoc(RedisSock *redis_sock, zval *z_result,

        if(line != NULL) {
            zval zv, *z = &zv;
-            if (redis_unserialize(redis_sock, line, line_len, z TSRMLS_CC)) {
+            if (redis_unpack(redis_sock, line, line_len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
                MAKE_STD_ZVAL(z);
                *z = zv;
--- a/common.h
+++ b/common.h
@@ -478,6 +478,7 @@ typedef enum _PUBSUB_TYPE {
 #define REDIS_OPT_READ_TIMEOUT       3
 #define REDIS_OPT_SCAN               4
 #define REDIS_OPT_FAILOVER           5
+#define REDIS_OPT_COMPRESSION        6

 /* cluster options */
 #define REDIS_FAILOVER_NONE              0
@@ -488,6 +489,9 @@ typedef enum _PUBSUB_TYPE {
 #define REDIS_SERIALIZER_NONE        0
 #define REDIS_SERIALIZER_PHP         1
 #define REDIS_SERIALIZER_IGBINARY    2
+/* compression */
+#define REDIS_COMPRESSION_NONE 0
+#define REDIS_COMPRESSION_LZF  1

 /* SCAN options */
 #define REDIS_SCAN_NORETRY 0
@@ -651,6 +655,7 @@ typedef struct {
    zend_string    *persistent_id;

    int            serializer;
+    int            compression;
    long           dbNumber;

    zend_string    *prefix;
--- a/config.m4
+++ b/config.m4
@@ -3,14 +3,16 @@ dnl config.m4 for extension redis

 PHP_ARG_ENABLE(redis, whether to enable redis support,
 dnl Make sure that the comment is aligned:
-[  --enable-redis           Enable redis support])
+[  --enable-redis          Enable redis support])

 PHP_ARG_ENABLE(redis-session, whether to enable sessions,
-[  --disable-redis-session      Disable session support], yes, no)
+[  --disable-redis-session Disable session support], yes, no)

 PHP_ARG_ENABLE(redis-igbinary, whether to enable igbinary serializer support,
-[  --enable-redis-igbinary      Enable igbinary serializer support], no, no)
+[  --enable-redis-igbinary Enable igbinary serializer support], no, no)

+PHP_ARG_ENABLE(redis-lzf, whether to enable lzf compression,
+[  --enable-redis-lzf      Enable lzf compression support], no, no)

 if test "$PHP_REDIS" != "no"; then

@@ -60,6 +62,13 @@ dnl Check for igbinary
    AC_MSG_RESULT([disabled])
  fi

+  if test "$PHP_REDIS_LZF" != "no"; then
+    PHP_ADD_INCLUDE(liblzf)
+    PHP_ADD_BUILD_DIR(liblzf)
+    lzf_sources="liblzf/lzf_c.c liblzf/lzf_d.c"
+    AC_DEFINE(HAVE_REDIS_LZF, 1, [ ])
+  fi
+
  dnl # --with-redis -> check with-path
  dnl SEARCH_PATH="/usr/local /usr"     # you might want to change this
  dnl SEARCH_FOR="/include/redis.h"  # you most likely want to change this
@@ -99,5 +108,5 @@ dnl Check for igbinary
  dnl
  dnl PHP_SUBST(REDIS_SHARED_LIBADD)

-  PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c, $ext_shared)
+  PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c $lzf_sources, $ext_shared)
 fi
--- a/1
+++ b/1
--- a/library.c
+++ b/library.c
@@ -11,6 +11,9 @@
 #ifdef HAVE_REDIS_IGBINARY
 #include "igbinary/igbinary.h"
 #endif
+#ifdef HAVE_REDIS_LZF
+#include "lzf.h"
+#endif
 #include <zend_exceptions.h>
 #include "php_redis.h"
 #include "library.h"
@@ -602,7 +605,7 @@ redis_spprintf(RedisSock *redis_sock, short *slot TSRMLS_DC, char **ret, char *k
                break;
            case 'v':
                arg.zv = va_arg(ap, zval*);
-                argfree = redis_serialize(redis_sock, arg.zv, &dup, &arglen TSRMLS_CC);
+                argfree = redis_pack(redis_sock, arg.zv, &dup, &arglen TSRMLS_CC);
                redis_cmd_append_sstr(&cmd, dup, arglen);
                if (argfree) efree(dup);
                break;
@@ -710,7 +713,7 @@ int redis_cmd_append_sstr_zval(smart_string *str, zval *z, RedisSock *redis_sock
    strlen_t vallen;
    int valfree, retval;

-    valfree = redis_serialize(redis_sock, z, &val, &vallen TSRMLS_CC);
+    valfree = redis_pack(redis_sock, z, &val, &vallen TSRMLS_CC);
    retval = redis_cmd_append_sstr(str, val, vallen);
    if (valfree) efree(val);

@@ -1248,7 +1251,7 @@ PHP_REDIS_API void redis_string_response(INTERNAL_FUNCTION_PARAMETERS, RedisSock
    }
    IF_NOT_ATOMIC() {
        zval zv, *z = &zv;
-        if (redis_unserialize(redis_sock, response, response_len, z TSRMLS_CC)) {
+        if (redis_unpack(redis_sock, response, response_len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
            MAKE_STD_ZVAL(z);
            *z = zv;
@@ -1258,7 +1261,7 @@ PHP_REDIS_API void redis_string_response(INTERNAL_FUNCTION_PARAMETERS, RedisSock
            add_next_index_stringl(z_tab, response, response_len);
        }
    } else {
-        if (!redis_unserialize(redis_sock, response, response_len, return_value TSRMLS_CC)) {
+        if (!redis_unpack(redis_sock, response, response_len, return_value TSRMLS_CC)) {
            RETVAL_STRINGL(response, response_len);
        }
    }
@@ -1386,6 +1389,7 @@ redis_sock_create(char *host, int host_len, unsigned short port,
    redis_sock->read_timeout = read_timeout;

    redis_sock->serializer = REDIS_SERIALIZER_NONE;
+    redis_sock->compression = REDIS_COMPRESSION_NONE;
    redis_sock->mode = ATOMIC;
    redis_sock->head = NULL;
    redis_sock->current = NULL;
@@ -1661,7 +1665,7 @@ redis_mbulk_reply_loop(INTERNAL_FUNCTION_PARAMETERS, RedisSock *redis_sock,
            (unserialize == UNSERIALIZE_VALS && i % 2 != 0)
        );
        zval zv, *z = &zv;
-        if (unwrap && redis_unserialize(redis_sock, line, len, z TSRMLS_CC)) {
+        if (unwrap && redis_unpack(redis_sock, line, len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
            MAKE_STD_ZVAL(z);
            *z = zv;
@@ -1709,7 +1713,7 @@ PHP_REDIS_API int redis_mbulk_reply_assoc(INTERNAL_FUNCTION_PARAMETERS, RedisSoc
        response = redis_sock_read(redis_sock, &response_len TSRMLS_CC);
        if(response != NULL) {
            zval zv0, *z = &zv0;
-            if (redis_unserialize(redis_sock, response, response_len, z TSRMLS_CC)) {
+            if (redis_unpack(redis_sock, response, response_len, z TSRMLS_CC)) {
 #if (PHP_MAJOR_VERSION < 7)
                MAKE_STD_ZVAL(z);
                *z = zv0;
@@ -1778,6 +1782,63 @@ PHP_REDIS_API void redis_free_socket(RedisSock *redis_sock)
    efree(redis_sock);
 }

+PHP_REDIS_API int
+redis_pack(RedisSock *redis_sock, zval *z, char **val, strlen_t *val_len TSRMLS_DC)
+{
+    char *buf, *data;
+    int valfree;
+    strlen_t len;
+    uint32_t res;
+
+    valfree = redis_serialize(redis_sock, z, &buf, &len TSRMLS_CC);
+    switch (redis_sock->compression) {
+        case REDIS_COMPRESSION_LZF:
+#ifdef HAVE_REDIS_LZF
+            data = emalloc(len);
+            res = lzf_compress(buf, len, data, len - 1);
+            if (res > 0 && res < len) {
+                if (valfree) efree(buf);
+                *val = data;
+                *val_len = res;
+                 return 1;
+            }
+            efree(data);
+#endif
+            break;
+    }
+    *val = buf;
+    *val_len = len;
+    return valfree;
+}
+
+PHP_REDIS_API int
+redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC)
+{
+    char *data;
+    int i, ret;
+    uint32_t res;
+
+    switch (redis_sock->compression) {
+        case REDIS_COMPRESSION_LZF:
+#ifdef HAVE_REDIS_LZF
+            errno = E2BIG;
+            for (i = 1; errno == E2BIG; ++i) {
+                data = emalloc(i * val_len);
+                if ((res = lzf_decompress(val, val_len, data, i * val_len)) == 0) {
+                    efree(data);
+                    continue;
+                } else if (redis_unserialize(redis_sock, data, res, z_ret TSRMLS_CC) == 0) {
+                    ZVAL_STRINGL(z_ret, data, res);
+                }
+                efree(data);
+                return 1;
+            }
+#endif
+            break;
+    }
+    return redis_unserialize(redis_sock, val, val_len, z_ret TSRMLS_CC);
+}
+
 PHP_REDIS_API int
 redis_serialize(RedisSock *redis_sock, zval *z, char **val, strlen_t *val_len
                TSRMLS_DC)
--- a/library.h
+++ b/library.h
@@ -76,6 +76,9 @@ redis_key_prefix(RedisSock *redis_sock, char **key, strlen_t *key_len);
 PHP_REDIS_API int
 redis_unserialize(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC);

+PHP_REDIS_API int redis_pack(RedisSock *redis_sock, zval *z, char **val, strlen_t *val_len TSRMLS_DC);
+PHP_REDIS_API int redis_unpack(RedisSock *redis_sock, const char *val, int val_len, zval *z_ret TSRMLS_DC);
+
 /*
 * Variant Read methods, mostly to implement eval
 */
--- a/package.xml
+++ b/package.xml
@@ -92,6 +92,9 @@ http://pear.php.net/dtd/package-2.0.xsd">
   <file role='src' name='crc16.h'/>
   <file role='src' name='redis_session.c'/>
   <file role='src' name='redis_session.h'/>
+   <file role='src' name='liblzf/lzf.h'/>
+   <file role='src' name='liblzf/lzf_c.c'/>
+   <file role='src' name='liblzf/lzf_d.c'/>
   <dir name='tests'>
     <file role='test' name='RedisArrayTest.php'/>
     <file role='test' name='RedisClusterTest.php'/>
@@ -117,6 +120,7 @@ http://pear.php.net/dtd/package-2.0.xsd">
 <providesextension>redis</providesextension>
 <extsrcrelease>
  <configureoption name="enable-redis-igbinary" prompt="enable igbinary serializer support?" default="no"/>
+  <configureoption name="enable-redis-lzf" prompt="enable lzf compression support?" default="no"/>
 </extsrcrelease>
 <changelog>
   <release>
--- a/redis.c
+++ b/redis.c
@@ -676,6 +676,7 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster TSRMLS_DC)
    zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SERIALIZER"), REDIS_OPT_SERIALIZER TSRMLS_CC);
    zend_declare_class_constant_long(ce, ZEND_STRL("OPT_PREFIX"), REDIS_OPT_PREFIX TSRMLS_CC);
    zend_declare_class_constant_long(ce, ZEND_STRL("OPT_READ_TIMEOUT"), REDIS_OPT_READ_TIMEOUT TSRMLS_CC);
+    zend_declare_class_constant_long(ce, ZEND_STRL("OPT_COMPRESSION"), REDIS_OPT_COMPRESSION TSRMLS_CC);

    /* serializer */
    zend_declare_class_constant_long(ce, ZEND_STRL("SERIALIZER_NONE"), REDIS_SERIALIZER_NONE TSRMLS_CC);
@@ -684,6 +685,12 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster TSRMLS_DC)
    zend_declare_class_constant_long(ce, ZEND_STRL("SERIALIZER_IGBINARY"), REDIS_SERIALIZER_IGBINARY TSRMLS_CC);
 #endif

+    /* compression */
+    zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_NONE"), REDIS_COMPRESSION_NONE TSRMLS_CC);
+#ifdef HAVE_REDIS_LZF
+    zend_declare_class_constant_long(ce, ZEND_STRL("COMPRESSION_LZF"), REDIS_COMPRESSION_LZF TSRMLS_CC);
+#endif
+
    /* scan options*/
    zend_declare_class_constant_long(ce, ZEND_STRL("OPT_SCAN"), REDIS_OPT_SCAN TSRMLS_CC);
    zend_declare_class_constant_long(ce, ZEND_STRL("SCAN_RETRY"), REDIS_SCAN_RETRY TSRMLS_CC);
@@ -806,6 +813,9 @@ PHP_MINFO_FUNCTION(redis)
    php_info_print_table_row(2, "Available serializers", "php, igbinary");
 #else
    php_info_print_table_row(2, "Available serializers", "php");
+#endif
+#ifdef HAVE_REDIS_LZF
+    php_info_print_table_row(2, "Available compression", "lzf");
 #endif
    php_info_print_table_end();
 }
--- a/redis_cluster.c
+++ b/redis_cluster.c
@@ -655,8 +655,7 @@ static int get_key_val_ht(redisCluster *c, HashTable *ht, HashPosition *ptr,
    }

    // Serialize our value if required
-    kv->val_free = redis_serialize(c->flags,z_val,&(kv->val),&(kv->val_len)
-        TSRMLS_CC);
+    kv->val_free = redis_pack(c->flags,z_val,&(kv->val),&(kv->val_len) TSRMLS_CC);

    // Success
    return 0;
--- a/redis_commands.c
+++ b/redis_commands.c
@@ -1007,7 +1007,7 @@ int redis_key_arr_cmd(INTERNAL_FUNCTION_PARAMETERS, RedisSock *redis_sock,

    /* Iterate our hash table, serializing and appending values */
    ZEND_HASH_FOREACH_VAL(ht_arr, z_val) {
-        val_free = redis_serialize(redis_sock, z_val, &val, &val_len TSRMLS_CC);
+        val_free = redis_pack(redis_sock, z_val, &val, &val_len TSRMLS_CC);
        redis_cmd_append_sstr(&cmdstr, val, val_len);
        if (val_free) efree(val);
    } ZEND_HASH_FOREACH_END();
@@ -1533,7 +1533,7 @@ int redis_hmset_cmd(INTERNAL_FUNCTION_PARAMETERS, RedisSock *redis_sock,
        }

        // Serialize value (if directed)
-        val_free = redis_serialize(redis_sock, z_val, &val, &val_len TSRMLS_CC);
+        val_free = redis_pack(redis_sock, z_val, &val, &val_len TSRMLS_CC);

        // Append the key and value to our command
        redis_cmd_append_sstr(&cmdstr, mem, mem_len);
@@ -1761,8 +1761,7 @@ static int redis_gen_pf_cmd(INTERNAL_FUNCTION_PARAMETERS, RedisSock *redis_sock,
                return FAILURE;
            }
        } else {
-            mem_free = redis_serialize(redis_sock, z_ele, &mem, &mem_len
-                TSRMLS_CC);
+            mem_free = redis_pack(redis_sock, z_ele, &mem, &mem_len TSRMLS_CC);

            zstr = NULL;
            if(!mem_free) {
@@ -2488,8 +2487,7 @@ int redis_zadd_cmd(INTERNAL_FUNCTION_PARAMETERS, RedisSock *redis_sock,
            redis_cmd_append_sstr_dbl(&cmdstr, zval_get_double(&z_args[i]));
        }
        // serialize value if requested
-        val_free = redis_serialize(redis_sock, &z_args[i+1], &val, &val_len
-            TSRMLS_CC);
+        val_free = redis_pack(redis_sock, &z_args[i+1], &val, &val_len TSRMLS_CC);
        redis_cmd_append_sstr(&cmdstr, val, val_len);

        // Free value if we serialized
@@ -3012,6 +3010,8 @@ void redis_getoption_handler(INTERNAL_FUNCTION_PARAMETERS,
    switch(option) {
        case REDIS_OPT_SERIALIZER:
            RETURN_LONG(redis_sock->serializer);
+        case REDIS_OPT_COMPRESSION:
+            RETURN_LONG(redis_sock->compression);
        case REDIS_OPT_PREFIX:
            if (redis_sock->prefix) {
                RETURN_STRINGL(ZSTR_VAL(redis_sock->prefix), ZSTR_LEN(redis_sock->prefix));
@@ -3055,6 +3055,17 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
                RETURN_TRUE;
            }
            break;
+        case REDIS_OPT_COMPRESSION:
+            val_long = atol(val_str);
+            if (val_long == REDIS_COMPRESSION_NONE
+#ifdef HAVE_REDIS_LZF
+                || val_long == REDIS_COMPRESSION_LZF
+#endif
+            ) {
+                redis_sock->compression = val_long;
+                RETURN_TRUE;
+            }
+            break;
        case REDIS_OPT_PREFIX:
            if (redis_sock->prefix) {
                zend_string_release(redis_sock->prefix);
--- a/tests/RedisTest.php
+++ b/tests/RedisTest.php
@@ -4256,6 +4256,24 @@ class Redis_Test extends TestSuite
        $this->assertTrue($this->redis->getOption(Redis::OPT_SERIALIZER) === Redis::SERIALIZER_NONE);       // get ok
    }

+    public function testCompressionLZF()
+    {
+        if (!defined('Redis::COMPRESSION_LZF')) {
+            $this->markTestSkipped();
+        }
+        $this->checkCompression(Redis::COMPRESSION_LZF);
+    }
+
+    private function checkCompression($mode)
+    {
+        $this->assertTrue($this->redis->setOption(Redis::OPT_COMPRESSION, $mode) === TRUE);  // set ok
+        $this->assertTrue($this->redis->getOption(Redis::OPT_COMPRESSION) === $mode);    // get ok
+
+        $val = 'xxxxxxxxxx';
+        $this->redis->set('key', $val);
+        $this->assertEquals($val, $this->redis->get('key'));
+    }
+
    public function testDumpRestore() {

        if (version_compare($this->version, "2.5.0", "lt")) {