php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

hash: Add MurmurHash3 with streaming support

Browse Source 
The implementation is based on the upstream PMurHash. The following
variants are implemented

- murmur3a, 32-bit hash
- murmur3c, 128-bit hash for x86
- murmur3f, 128-bit hash for x64

The custom seed support is not targeted by this implementation. It will
need a major change to the API, so then custom arguments can be passed
through `hash_init`. For now, the starting hash is always zero.

Fixes bug #68109, closes #6059

Signed-off-by: Anatol Belski <ab@php.net>
Co-Developed-by: Michael Wallner <mike@php.net>
Signed-off-by: Michael Wallner <mike@php.net>
This commit is contained in:
Anatol Belski
2020-08-30 14:14:04 +02:00
parent 831abe2d90
commit 72e91e9fc8
17 changed files with 1392 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ext/hash/config.m4
View File

@@ -35,11 +35,12 @@ fi
EXT_HASH_SOURCES="hash.c hash_md.c hash_sha.c hash_ripemd.c hash_haval.c \
hash_tiger.c hash_gost.c hash_snefru.c hash_whirlpool.c hash_adler32.c \
hash_crc32.c hash_fnv.c hash_joaat.c $EXT_HASH_SHA3_SOURCES"
hash_crc32.c hash_fnv.c hash_joaat.c $EXT_HASH_SHA3_SOURCES
murmur/PMurHash.c murmur/PMurHash128.c hash_murmur.c"
EXT_HASH_HEADERS="php_hash.h php_hash_md.h php_hash_sha.h php_hash_ripemd.h \
php_hash_haval.h php_hash_tiger.h php_hash_gost.h php_hash_snefru.h \
php_hash_whirlpool.h php_hash_adler32.h php_hash_crc32.h \
php_hash_fnv.h php_hash_joaat.h php_hash_sha3.h"
php_hash_fnv.h php_hash_joaat.h php_hash_sha3.h php_hash_murmur.h"
PHP_NEW_EXTENSION(hash, $EXT_HASH_SOURCES, 0,,$PHP_HASH_CFLAGS)
PHP_INSTALL_HEADERS(ext/hash, $EXT_HASH_HEADERS)

11
ext/hash/config.w32
View File

@@ -11,7 +11,7 @@ PHP_HASH = 'yes';
EXTENSION('hash', 'hash.c hash_md.c hash_sha.c hash_ripemd.c hash_haval.c ' +
'hash_tiger.c hash_gost.c hash_snefru.c hash_whirlpool.c ' +
'hash_adler32.c hash_crc32.c hash_joaat.c hash_fnv.c ' +
'hash_sha3.c', false);
'hash_sha3.c hash_murmur.c', false);
var hash_sha3_dir = 'ext/hash/sha3/generic' + (X64 ? '64' : '32') + 'lc';
@@ -28,7 +28,14 @@ if (!CHECK_HEADER_ADD_INCLUDE('KeccakHash.h', 'CFLAGS_HASH', hash_sha3_dir)) {
ADD_FLAG('CFLAGS_HASH', '/DKeccakP200_excluded /DKeccakP400_excluded /DKeccakP800_excluded /DZEND_ENABLE_STATIC_TSRMLS_CACHE=1');
var hash_murmur_dir = 'ext/hash/murmur';
if (!CHECK_HEADER_ADD_INCLUDE('PMurHash.h', 'CFLAGS_HASH', hash_murmur_dir)) {
ERROR('Unable to locate murmur headers');
}
ADD_SOURCES(hash_murmur_dir, 'PMurHash.c PMurHash128.c', 'hash');
PHP_INSTALL_HEADERS('ext/hash/', 'php_hash.h php_hash_md.h php_hash_sha.h ' +
'php_hash_ripemd.h php_hash_haval.h php_hash_tiger.h ' +
'php_hash_gost.h php_hash_snefru.h php_hash_whirlpool.h ' +
'php_hash_adler32.h php_hash_crc32.h php_hash_sha3.h');
'php_hash_adler32.h php_hash_crc32.h php_hash_sha3.h ' +
'php_hash_murmur.h');

8
ext/hash/hash.c
View File

@@ -52,7 +52,7 @@ struct mhash_bc_entry {
int value;
};
#define MHASH_NUM_ALGOS 35
#define MHASH_NUM_ALGOS 38
static struct mhash_bc_entry mhash_to_hash[MHASH_NUM_ALGOS] = {
{"CRC32", "crc32", 0}, /* used by bzip */
@@ -90,6 +90,9 @@ static struct mhash_bc_entry mhash_to_hash[MHASH_NUM_ALGOS] = {
{"FNV1A64", "fnv1a64", 32},
{"JOAAT", "joaat", 33},
{"CRC32C", "crc32c", 34}, /* Castagnoli's CRC, used by iSCSI, SCTP, Btrfs, ext4, etc */
{"MURMUR3A", "murmur3a", 35},
{"MURMUR3C", "murmur3c", 36},
{"MURMUR3F", "murmur3f", 37},
};
#endif
@@ -1586,6 +1589,9 @@ PHP_MINIT_FUNCTION(hash)
php_hash_register_algo("fnv164", &php_hash_fnv164_ops);
php_hash_register_algo("fnv1a64", &php_hash_fnv1a64_ops);
php_hash_register_algo("joaat", &php_hash_joaat_ops);
php_hash_register_algo("murmur3a", &php_hash_murmur3a_ops);
php_hash_register_algo("murmur3c", &php_hash_murmur3c_ops);
php_hash_register_algo("murmur3f", &php_hash_murmur3f_ops);
PHP_HASH_HAVAL_REGISTER(3,128);
PHP_HASH_HAVAL_REGISTER(3,160);

186
ext/hash/hash_murmur.c Normal file
View File

@@ -0,0 +1,186 @@
/*
+----------------------------------------------------------------------+
| Copyright (c) The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| http://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
| Author: Anatol Belski <ab@php.net> |
+----------------------------------------------------------------------+
*/
#include "php_hash.h"
#include "php_hash_murmur.h"
#include "murmur/PMurHash.h"
#include "murmur/PMurHash128.h"
const php_hash_ops php_hash_murmur3a_ops = {
"murmur3a",
(php_hash_init_func_t) PHP_MURMUR3AInit,
(php_hash_update_func_t) PHP_MURMUR3AUpdate,
(php_hash_final_func_t) PHP_MURMUR3AFinal,
(php_hash_copy_func_t) PHP_MURMUR3ACopy,
php_hash_serialize,
php_hash_unserialize,
PHP_MURMUR3A_SPEC,
4,
4,
sizeof(PHP_MURMUR3A_CTX),
0
};
PHP_HASH_API void PHP_MURMUR3AInit(PHP_MURMUR3A_CTX *ctx)
{
ctx->h = 0;
ctx->carry = 0;
ctx->len = 0;
}
PHP_HASH_API void PHP_MURMUR3AUpdate(PHP_MURMUR3A_CTX *ctx, const unsigned char *in, size_t len)
{
ctx->len += len;
PMurHash32_Process(&ctx->h, &ctx->carry, in, len);
}
PHP_HASH_API void PHP_MURMUR3AFinal(unsigned char digest[4], PHP_MURMUR3A_CTX *ctx)
{
ctx->h = PMurHash32_Result(ctx->h, ctx->carry, ctx->len);
digest[0] = (unsigned char)((ctx->h >> 24) & 0xff);
digest[1] = (unsigned char)((ctx->h >> 16) & 0xff);
digest[2] = (unsigned char)((ctx->h >> 8) & 0xff);
digest[3] = (unsigned char)(ctx->h & 0xff);
}
PHP_HASH_API int PHP_MURMUR3ACopy(const php_hash_ops *ops, PHP_MURMUR3A_CTX *orig_context, PHP_MURMUR3A_CTX *copy_context)
{
copy_context->h = orig_context->h;
copy_context->carry = orig_context->carry;
copy_context->len = orig_context->len;
return SUCCESS;
}
const php_hash_ops php_hash_murmur3c_ops = {
"murmur3c",
(php_hash_init_func_t) PHP_MURMUR3CInit,
(php_hash_update_func_t) PHP_MURMUR3CUpdate,
(php_hash_final_func_t) PHP_MURMUR3CFinal,
(php_hash_copy_func_t) PHP_MURMUR3CCopy,
php_hash_serialize,
php_hash_unserialize,
PHP_MURMUR3C_SPEC,
16,
4,
sizeof(PHP_MURMUR3C_CTX),
0
};
PHP_HASH_API void PHP_MURMUR3CInit(PHP_MURMUR3C_CTX *ctx)
{
memset(&ctx->h, 0, sizeof ctx->h);
memset(&ctx->carry, 0, sizeof ctx->carry);
ctx->len = 0;
}
PHP_HASH_API void PHP_MURMUR3CUpdate(PHP_MURMUR3C_CTX *ctx, const unsigned char *in, size_t len)
{
ctx->len += len;
PMurHash128x86_Process(ctx->h, ctx->carry, in, len);
}
PHP_HASH_API void PHP_MURMUR3CFinal(unsigned char digest[4], PHP_MURMUR3C_CTX *ctx)
{
uint32_t h[4] = {0};
PMurHash128x86_Result(ctx->h, ctx->carry, ctx->len, h);
digest[0] = (unsigned char)((h[0] >> 24) & 0xff);
digest[1] = (unsigned char)((h[0] >> 16) & 0xff);
digest[2] = (unsigned char)((h[0] >> 8) & 0xff);
digest[3] = (unsigned char)(h[0] & 0xff);
digest[4] = (unsigned char)((h[1] >> 24) & 0xff);
digest[5] = (unsigned char)((h[1] >> 16) & 0xff);
digest[6] = (unsigned char)((h[1] >> 8) & 0xff);
digest[7] = (unsigned char)(h[1] & 0xff);
digest[8] = (unsigned char)((h[2] >> 24) & 0xff);
digest[9] = (unsigned char)((h[2] >> 16) & 0xff);
digest[10] = (unsigned char)((h[2] >> 8) & 0xff);
digest[11] = (unsigned char)(h[2] & 0xff);
digest[12] = (unsigned char)((h[3] >> 24) & 0xff);
digest[13] = (unsigned char)((h[3] >> 16) & 0xff);
digest[14] = (unsigned char)((h[3] >> 8) & 0xff);
digest[15] = (unsigned char)(h[3] & 0xff);
}
PHP_HASH_API int PHP_MURMUR3CCopy(const php_hash_ops *ops, PHP_MURMUR3C_CTX *orig_context, PHP_MURMUR3C_CTX *copy_context)
{
memcpy(&copy_context->h, &orig_context->h, sizeof orig_context->h);
memcpy(&copy_context->carry, &orig_context->carry, sizeof orig_context->carry);
copy_context->len = orig_context->len;
return SUCCESS;
}
const php_hash_ops php_hash_murmur3f_ops = {
"murmur3f",
(php_hash_init_func_t) PHP_MURMUR3FInit,
(php_hash_update_func_t) PHP_MURMUR3FUpdate,
(php_hash_final_func_t) PHP_MURMUR3FFinal,
(php_hash_copy_func_t) PHP_MURMUR3FCopy,
php_hash_serialize,
php_hash_unserialize,
PHP_MURMUR3F_SPEC,
16,
8,
sizeof(PHP_MURMUR3F_CTX),
0
};
PHP_HASH_API void PHP_MURMUR3FInit(PHP_MURMUR3F_CTX *ctx)
{
memset(&ctx->h, 0, sizeof ctx->h);
memset(&ctx->carry, 0, sizeof ctx->carry);
ctx->len = 0;
}
PHP_HASH_API void PHP_MURMUR3FUpdate(PHP_MURMUR3F_CTX *ctx, const unsigned char *in, size_t len)
{
ctx->len += len;
PMurHash128x64_Process(ctx->h, ctx->carry, in, len);
}
PHP_HASH_API void PHP_MURMUR3FFinal(unsigned char digest[4], PHP_MURMUR3F_CTX *ctx)
{
uint64_t h[2] = {0};
PMurHash128x64_Result(ctx->h, ctx->carry, ctx->len, h);
digest[0] = (unsigned char)((h[0] >> 56) & 0xff);
digest[1] = (unsigned char)((h[0] >> 48) & 0xff);
digest[2] = (unsigned char)((h[0] >> 40) & 0xff);
digest[3] = (unsigned char)((h[0] >> 32) & 0xff);
digest[4] = (unsigned char)((h[0] >> 24) & 0xff);
digest[5] = (unsigned char)((h[0] >> 16) & 0xff);
digest[6] = (unsigned char)((h[0] >> 8) & 0xff);
digest[7] = (unsigned char)(h[0] & 0xff);
digest[8] = (unsigned char)((h[1] >> 56) & 0xff);
digest[9] = (unsigned char)((h[1] >> 48) & 0xff);
digest[10] = (unsigned char)((h[1] >> 40) & 0xff);
digest[11] = (unsigned char)((h[1] >> 32) & 0xff);
digest[12] = (unsigned char)((h[1] >> 24) & 0xff);
digest[13] = (unsigned char)((h[1] >> 16) & 0xff);
digest[14] = (unsigned char)((h[1] >> 8) & 0xff);
digest[15] = (unsigned char)(h[1] & 0xff);
}
PHP_HASH_API int PHP_MURMUR3FCopy(const php_hash_ops *ops, PHP_MURMUR3F_CTX *orig_context, PHP_MURMUR3F_CTX *copy_context)
{
memcpy(&copy_context->h, &orig_context->h, sizeof orig_context->h);
memcpy(&copy_context->carry, &orig_context->carry, sizeof orig_context->carry);
copy_context->len = orig_context->len;
return SUCCESS;
}

226
ext/hash/murmur/PMurHash.c Normal file
View File

@@ -0,0 +1,226 @@
/*-----------------------------------------------------------------------------
* MurmurHash3 was written by Austin Appleby, and is placed in the public
* domain.
*
* This implementation was written by Shane Day, and is also public domain.
*
* This is a portable ANSI C implementation of MurmurHash3_x86_32 (Murmur3A)
* with support for progressive processing.
*/
/*-----------------------------------------------------------------------------
If you want to understand the MurmurHash algorithm you would be much better
off reading the original source. Just point your browser at:
http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
What this version provides?
1. Progressive data feeding. Useful when the entire payload to be hashed
does not fit in memory or when the data is streamed through the application.
Also useful when hashing a number of strings with a common prefix. A partial
hash of a prefix string can be generated and reused for each suffix string.
How does it work?
We can only process entire 32 bit chunks of input, except for the very end
that may be shorter. So along with the partial hash we need to give back to
the caller a carry containing up to 3 bytes that we were unable to process.
This carry also needs to record the number of bytes the carry holds. I use
the low 2 bits as a count (0..3) and the carry bytes are shifted into the
high byte in stream order.
To handle endianess I simply use a macro that reads a uint32_t and define
that macro to be a direct read on little endian machines, a read and swap
on big endian machines, or a byte-by-byte read if the endianess is unknown.
-----------------------------------------------------------------------------*/
#include "PMurHash.h"
// /* MSVC warnings we choose to ignore */
// #if defined(_MSC_VER)
// #pragma warning(disable: 4127) /* conditional expression is constant */
// #endif
/*-----------------------------------------------------------------------------
* Endianess, misalignment capabilities and util macros
*
* The following 3 macros are defined in this section. The other macros defined
* are only needed to help derive these 3.
*
* READ_UINT32(x) Read a little endian unsigned 32-bit int
* UNALIGNED_SAFE Defined if READ_UINT32 works on non-word boundaries
* ROTL32(x,r) Rotate x left by r bits
*/
/* I386 or AMD64 */
#if defined(_M_I86) || defined(_M_IX86) || defined(_X86_) || defined(__i386__) || defined(__i386) || defined(i386) \
|| defined(_M_X64) || defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64)
#define UNALIGNED_SAFE
#endif
/* I386 or AMD64 */
#if defined(_M_I86) || defined(_M_IX86) || defined(_X86_) || defined(__i386__) || defined(__i386) || defined(i386) \
|| defined(_M_X64) || defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64)
#define UNALIGNED_SAFE
#endif
/* Find best way to ROTL */
#if defined(_MSC_VER)
#define FORCE_INLINE static __forceinline
#include <stdlib.h> /* Microsoft put _rotl declaration in here */
#define ROTL32(x,y) _rotl(x,y)
#else
#define FORCE_INLINE static inline __attribute__((always_inline))
/* gcc recognises this code and generates a rotate instruction for CPUs with one */
#define ROTL32(x,r) (((uint32_t)x << r) | ((uint32_t)x >> (32 - r)))
#endif
#include "endianness.h"
#define READ_UINT32(ptr) getblock32((uint32_t *)ptr, 0)
/*-----------------------------------------------------------------------------
* Core murmurhash algorithm macros */
static const uint32_t kC1 = 0xcc9e2d51;
static const uint32_t kC2 = 0x1b873593;
/* This is the main processing body of the algorithm. It operates
* on each full 32-bits of input. */
#define doblock(h1, k1) \
do {\
k1 *= kC1;\
k1 = ROTL32(k1,15);\
k1 *= kC2;\
\
h1 ^= k1;\
h1 = ROTL32(h1,13);\
h1 = h1*5+0xe6546b64;\
} while(0)
/* Append unaligned bytes to carry, forcing hash churn if we have 4 bytes */
/* cnt=bytes to process, h1=name of h1 var, c=carry, n=bytes in c, ptr/len=payload */
#define dobytes(cnt, h1, c, n, ptr, len) \
do {\
unsigned __cnt = cnt;\
while(__cnt--) {\
c = c>>8 | (uint32_t)*ptr++<<24;\
n++; len--;\
if(n==4) {\
doblock(h1, c);\
n = 0;\
}\
}\
} while(0)
/*---------------------------------------------------------------------------*/
/* Main hashing function. Initialise carry to 0 and h1 to 0 or an initial seed
* if wanted. Both ph1 and pcarry are required arguments. */
void PMurHash32_Process(uint32_t *ph1, uint32_t *pcarry, const void *key, int len)
{
uint32_t h1 = *ph1;
uint32_t c = *pcarry;
const uint8_t *ptr = (uint8_t*)key;
const uint8_t *end;
/* Extract carry count from low 2 bits of c value */
int n = c & 3;
#if defined(UNALIGNED_SAFE)
/* This CPU handles unaligned word access */
// #pragma message ( "UNALIGNED_SAFE" )
/* Consume any carry bytes */
int i = (4-n) & 3;
if(i && i <= len) {
dobytes(i, h1, c, n, ptr, len);
}
/* Process 32-bit chunks */
end = ptr + (len & ~3);
for( ; ptr < end ; ptr+=4) {
uint32_t k1 = READ_UINT32(ptr);
doblock(h1, k1);
}
#else /*UNALIGNED_SAFE*/
/* This CPU does not handle unaligned word access */
// #pragma message ( "ALIGNED" )
/* Consume enough so that the next data byte is word aligned */
int i = -(intptr_t)(void *)ptr & 3;
if(i && i <= len) {
dobytes(i, h1, c, n, ptr, len);
}
/* We're now aligned. Process in aligned blocks. Specialise for each possible carry count */
end = ptr + (len & ~3);
switch(n) { /* how many bytes in c */
case 0: /* c=[----] w=[3210] b=[3210]=w c'=[----] */
for( ; ptr < end ; ptr+=4) {
uint32_t k1 = READ_UINT32(ptr);
doblock(h1, k1);
}
break;
case 1: /* c=[0---] w=[4321] b=[3210]=c>>24|w<<8 c'=[4---] */
for( ; ptr < end ; ptr+=4) {
uint32_t k1 = c>>24;
c = READ_UINT32(ptr);
k1 |= c<<8;
doblock(h1, k1);
}
break;
case 2: /* c=[10--] w=[5432] b=[3210]=c>>16|w<<16 c'=[54--] */
for( ; ptr < end ; ptr+=4) {
uint32_t k1 = c>>16;
c = READ_UINT32(ptr);
k1 |= c<<16;
doblock(h1, k1);
}
break;
case 3: /* c=[210-] w=[6543] b=[3210]=c>>8|w<<24 c'=[654-] */
for( ; ptr < end ; ptr+=4) {
uint32_t k1 = c>>8;
c = READ_UINT32(ptr);
k1 |= c<<24;
doblock(h1, k1);
}
}
#endif /*UNALIGNED_SAFE*/
/* Advance over whole 32-bit chunks, possibly leaving 1..3 bytes */
len -= len & ~3;
/* Append any remaining bytes into carry */
dobytes(len, h1, c, n, ptr, len);
/* Copy out new running hash and carry */
*ph1 = h1;
*pcarry = (c & ~0xff) | n;
}
/*---------------------------------------------------------------------------*/
/* Finalize a hash. To match the original Murmur3A the total_length must be provided */
uint32_t PMurHash32_Result(uint32_t h, uint32_t carry, uint32_t total_length)
{
uint32_t k1;
int n = carry & 3;
if(n) {
k1 = carry >> (4-n)*8;
k1 *= kC1; k1 = ROTL32(k1,15); k1 *= kC2; h ^= k1;
}
h ^= total_length;
/* fmix */
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
}

31
ext/hash/murmur/PMurHash.h Normal file
View File

@@ -0,0 +1,31 @@
/*-----------------------------------------------------------------------------
* MurmurHash3 was written by Austin Appleby, and is placed in the public
* domain.
*
* This implementation was written by Shane Day, and is also public domain.
*
* This implementation was modified to match PMurHash128.cpp.
*/
/* ------------------------------------------------------------------------- */
// Microsoft Visual Studio
#if defined(_MSC_VER) && (_MSC_VER < 1600)
typedef unsigned char uint8_t;
typedef unsigned int uint32_t;
// Other compilers
#else // defined(_MSC_VER)
#include <stdint.h>
#endif // !defined(_MSC_VER)
/* ------------------------------------------------------------------------- */
/* Prototypes */
void PMurHash32_Process(uint32_t *ph1, uint32_t *pcarry, const void *key, int len);
uint32_t PMurHash32_Result(uint32_t h1, uint32_t carry, uint32_t total_length);

640
ext/hash/murmur/PMurHash128.c Normal file
View File

@@ -0,0 +1,640 @@
/*-----------------------------------------------------------------------------
* MurmurHash3 was written by Austin Appleby, and is placed in the public
* domain.
*
* This is a c++ implementation of MurmurHash3_128 with support for progressive
* processing based on PMurHash implementation written by Shane Day.
*/
/*-----------------------------------------------------------------------------
If you want to understand the MurmurHash algorithm you would be much better
off reading the original source. Just point your browser at:
http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
What this version provides?
1. Progressive data feeding. Useful when the entire payload to be hashed
does not fit in memory or when the data is streamed through the application.
Also useful when hashing a number of strings with a common prefix. A partial
hash of a prefix string can be generated and reused for each suffix string.
How does it work?
We can only process entire 128 bit chunks of input, except for the very end
that may be shorter. So along with the partial hash we need to give back to
the caller a carry containing up to 15 bytes that we were unable to process.
This carry also needs to record the number of bytes the carry holds. I use
the low 4 bits as a count (0..15) and the carry bytes are shifted into the
high byte in stream order.
To handle endianess I simply use a macro that reads an uint and define
that macro to be a direct read on little endian machines, a read and swap
on big endian machines.
-----------------------------------------------------------------------------*/
#include "PMurHash128.h"
/*-----------------------------------------------------------------------------
* Endianess, misalignment capabilities and util macros
*
* The following 5 macros are defined in this section. The other macros defined
* are only needed to help derive these 5.
*
* READ_UINT32(x,i) Read a little endian unsigned 32-bit int at index
* READ_UINT64(x,i) Read a little endian unsigned 64-bit int at index
* UNALIGNED_SAFE Defined if READ_UINTXX works on non-word boundaries
* ROTL32(x,r) Rotate x left by r bits
* ROTL64(x,r) Rotate x left by r bits
* BIG_CONSTANT
* FORCE_INLINE
*/
/* I386 or AMD64 */
#if defined(_M_I86) || defined(_M_IX86) || defined(_X86_) || defined(__i386__) || defined(__i386) || defined(i386) \
|| defined(_M_X64) || defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64)
#define UNALIGNED_SAFE
#endif
/* Find best way to ROTL */
#if defined(_MSC_VER)
#define FORCE_INLINE static __forceinline
#include <stdlib.h> /* Microsoft put _rotl declaration in here */
#define ROTL32(x,y) _rotl(x,y)
#define ROTL64(x,y) _rotl64(x,y)
#define BIG_CONSTANT(x) (x)
#else
#define FORCE_INLINE static inline __attribute__((always_inline))
/* gcc recognises this code and generates a rotate instruction for CPUs with one */
#define ROTL32(x,r) (((uint32_t)x << r) | ((uint32_t)x >> (32 - r)))
#define ROTL64(x,r) (((uint64_t)x << r) | ((uint64_t)x >> (64 - r)))
#define BIG_CONSTANT(x) (x##LLU)
#endif
#include "endianness.h"
#define READ_UINT64(ptr,i) getblock64((uint64_t *)ptr,i)
#define READ_UINT32(ptr,i) getblock32((uint32_t *)ptr,i)
//-----------------------------------------------------------------------------
// Finalization mix - force all bits of a hash block to avalanche
FORCE_INLINE uint32_t fmix32 ( uint32_t h )
{
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
}
//----------
FORCE_INLINE uint64_t fmix64 ( uint64_t k )
{
k ^= k >> 33;
k *= BIG_CONSTANT(0xff51afd7ed558ccd);
k ^= k >> 33;
k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
k ^= k >> 33;
return k;
}
/*-----------------------------------------------------------------------------*
PMurHash128x86
*-----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
* Core murmurhash algorithm macros */
static const uint32_t kC1 = 0x239b961b;
static const uint32_t kC2 = 0xab0e9789;
static const uint32_t kC3 = 0x38b34ae5;
static const uint32_t kC4 = 0xa1e38b93;
/* This is the main processing body of the algorithm. It operates
* on each full 128-bits of input. */
#define doblock128x86(h1, h2, h3, h4, k1, k2, k3,k4)\
do {\
k1 *= kC1; k1 = ROTL32(k1,15); k1 *= kC2; h1 ^= k1;\
\
h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;\
\
k2 *= kC2; k2 = ROTL32(k2,16); k2 *= kC3; h2 ^= k2;\
\
h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;\
\
k3 *= kC3; k3 = ROTL32(k3,17); k3 *= kC4; h3 ^= k3;\
\
h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;\
\
k4 *= kC4; k4 = ROTL32(k4,18); k4 *= kC1; h4 ^= k4;\
\
h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;\
} while(0)
/* Append unaligned bytes to carry, forcing hash churn if we have 16 bytes */
/* cnt=bytes to process, h1-h4=hash k1-k4=carry, n=bytes in carry, ptr/len=payload */
#define dobytes128x86(cnt, h1, h2, h3, h4, k1, k2, k3, k4, n, ptr, len)\
do {\
unsigned __cnt = cnt;\
for(;__cnt--; len--) {\
switch(n) {\
case 0: case 1: case 2: case 3:\
k1 = k1>>8 | (uint32_t)*ptr++<<24;\
++n; break;\
\
case 4: case 5: case 6: case 7:\
k2 = k2>>8 | (uint32_t)*ptr++<<24;\
++n; break;\
\
case 8: case 9: case 10: case 11:\
k3 = k3>>8 | (uint32_t)*ptr++<<24;\
++n; break;\
\
case 12: case 13: case 14:\
k4 = k4>>8 | (uint32_t)*ptr++<<24;\
++n; break;\
\
case 15:\
k4 = k4>>8 | (uint32_t)*ptr++<<24;\
doblock128x86(h1, h2, h3, h4, k1, k2, k3, k4);\
n = 0; break;\
}\
}\
} while(0)
/* Finalize a hash. To match the original Murmur3_128x86 the total_length must be provided */
void PMurHash128x86_Result(const uint32_t *ph, const uint32_t *pcarry, uint32_t total_length, uint32_t *out)
{
uint32_t h1 = ph[0];
uint32_t h2 = ph[1];
uint32_t h3 = ph[2];
uint32_t h4 = ph[3];
uint32_t k1, k2, k3, k4 = pcarry[3];
int n = k4 & 15;
switch(n) {
case 1: case 2: case 3: case 4:
k1 = pcarry[0] >> (4-n)*8;
goto finrot_k1;
case 5: case 6: case 7: case 8:
k2 = pcarry[1] >> (8-n)*8;
goto finrot_k21;
case 9: case 10: case 11: case 12:
k3 = pcarry[2] >> (12-n)*8;
goto finrot_k321;
case 13: case 14: case 15:
k4 >>= (16-n)*8;
goto finrot_k4321;
default:
goto skiprot;
}
finrot_k4321:
k4 *= kC4; k4 = ROTL32(k4,18); k4 *= kC1; h4 ^= k4;
k3 = pcarry[2];
finrot_k321:
k3 *= kC3; k3 = ROTL32(k3,17); k3 *= kC4; h3 ^= k3;
k2 = pcarry[1];
finrot_k21:
k2 *= kC2; k2 = ROTL32(k2,16); k2 *= kC3; h2 ^= k2;
k1 = pcarry[0];
finrot_k1:
k1 *= kC1; k1 = ROTL32(k1,15); k1 *= kC2; h1 ^= k1;
skiprot:
//----------
// finalization
h1 ^= total_length; h2 ^= total_length;
h3 ^= total_length; h4 ^= total_length;
h1 += h2; h1 += h3; h1 += h4;
h2 += h1; h3 += h1; h4 += h1;
h1 = fmix32(h1);
h2 = fmix32(h2);
h3 = fmix32(h3);
h4 = fmix32(h4);
h1 += h2; h1 += h3; h1 += h4;
h2 += h1; h3 += h1; h4 += h1;
out[0] = h1;
out[1] = h2;
out[2] = h3;
out[3] = h4;
}
/*---------------------------------------------------------------------------*/
/* Main hashing function. Initialise carry[4] to {0,0,0,0} and h[4] to an initial {seed,seed,seed,seed}
* if wanted. Both ph and pcarry are required arguments. */
void PMurHash128x86_Process(uint32_t * const ph, uint32_t * const pcarry, const void * const key, int len)
{
uint32_t h1 = ph[0];
uint32_t h2 = ph[1];
uint32_t h3 = ph[2];
uint32_t h4 = ph[3];
uint32_t k1 = pcarry[0];
uint32_t k2 = pcarry[1];
uint32_t k3 = pcarry[2];
uint32_t k4 = pcarry[3];
const uint8_t *ptr = (uint8_t*)key;
const uint8_t *end;
/* Extract carry count from low 4 bits of c value */
int n = k4 & 15;
#if defined(UNALIGNED_SAFE)
/* This CPU handles unaligned word access */
// #pragma message ( "UNALIGNED_SAFE" )
/* Consume any carry bytes */
int i = (16-n) & 15;
if(i && i <= len) {
dobytes128x86(i, h1, h2, h3, h4, k1, k2, k3, k4, n, ptr, len);
}
/* Process 128-bit chunks */
end = ptr + (len & ~15);
for( ; ptr < end ; ptr+=16) {
k1 = READ_UINT32(ptr, 0);
k2 = READ_UINT32(ptr, 1);
k3 = READ_UINT32(ptr, 2);
k4 = READ_UINT32(ptr, 3);
doblock128x86(h1, h2, h3, h4, k1, k2, k3, k4);
}
#else /*UNALIGNED_SAFE*/
/* This CPU does not handle unaligned word access */
// #pragma message ( "ALIGNED" )
/* Consume enough so that the next data byte is word aligned */
int i = -(intptr_t)(void *)ptr & 3;
if(i && i <= len) {
dobytes128x86(i, h1, h2, h3, h4, k1, k2, k3, k4, n, ptr, len);
}
/* We're now aligned. Process in aligned blocks. Specialise for each possible carry count */
end = ptr + (len & ~15);
switch(n) { /* how many bytes in c */
case 0: /*
k1=[----] k2=[----] k2=[----] k4=[----] w=[3210 7654 ba98 fedc] b=[3210 7654 ba98 fedc] */
for( ; ptr < end ; ptr+=16) {
k1 = READ_UINT32(ptr, 0);
k2 = READ_UINT32(ptr, 1);
k3 = READ_UINT32(ptr, 2);
k4 = READ_UINT32(ptr, 3);
doblock128x86(h1, h2, h3, h4, k1, k2, k3, k4);
}
break;
case 1: case 2: case 3: /*
k1=[10--] k2=[----] k3=[----] k4=[----] w=[5432 9876 dcba hgfe] b=[3210 7654 ba98 fedc] k1'=[hg--] */
{
const int lshift = n*8, rshift = 32-lshift;
for( ; ptr < end ; ptr+=16) {
uint32_t c = k1>>rshift; // --10
k2 = READ_UINT32(ptr, 0); // 5432
c |= k2<<lshift; // 3210.
k1 = READ_UINT32(ptr, 1); // 9876
k2 = k1<<lshift | k2>>rshift; // 7654.
k4 = READ_UINT32(ptr, 2); // dcba
k3 = k4<<lshift | k1>>rshift; // ba98.
k1 = READ_UINT32(ptr, 3); // hgfe.
k4 = k1<<lshift | k4>>rshift; // fedc.
doblock128x86(h1, h2, h3, h4, c, k2, k3, k4);
}
}
break;
case 4: /*
k1=[3210] k2=[----] k3=[----] k4=[----] w=[7654 ba98 fedc jihg] b=[3210 7654 ba98 fedc] k1'=[jihg] */
for( ; ptr < end ; ptr+=16) {
k2 = READ_UINT32(ptr, 0);
k3 = READ_UINT32(ptr, 1);
k4 = READ_UINT32(ptr, 2);
doblock128x86(h1, h2, h3, h4, k1, k2, k3, k4);
k1 = READ_UINT32(ptr, 3);
}
break;
case 5: case 6: case 7: /*
k1=[3210] k2=[54--] k3=[----] k4=[----] w=[9876 dcba hgfe lkji] b=[3210 7654 ba98 fedc] k1'=[jihg] k2'=[lk--] */
{
const int lshift = n*8-32, rshift = 32-lshift;
for( ; ptr < end ; ptr+=16) {
uint32_t c = k2>>rshift; // --54
k3 = READ_UINT32(ptr, 0); // 9876
c |= k3<<lshift; // 7654.
k4 = READ_UINT32(ptr, 1); // dcba
k3 = k4<<lshift | k3>>rshift; // ba98.
k2 = READ_UINT32(ptr, 2); // hgfe
k4 = k2<<lshift | k4>>rshift; // fedc.
doblock128x86(h1, h2, h3, h4, k1, c, k3, k4);
k1 = k2>>rshift; // --hg
k2 = READ_UINT32(ptr, 3); // lkji.
k1 |= k2<<lshift; // jihg.
}
}
case 8: /*
k1=[3210] k2=[7654] k3=[----] k4=[----] w=[ba98 fedc jihg nmlk] b=[3210 7654 ba98 fedc] k1'=[jihg] k2'=[nmlk] */
for( ; ptr < end ; ptr+=16) {
k3 = READ_UINT32(ptr, 0);
k4 = READ_UINT32(ptr, 1);
doblock128x86(h1, h2, h3, h4, k1, k2, k3, k4);
k1 = READ_UINT32(ptr, 2);
k2 = READ_UINT32(ptr, 3);
}
break;
case 9: case 10: case 11: /*
k1=[3210] k2=[7654] k3=[98--] k4=[----] w=[dcba hgfe lkji ponm] b=[3210 7654 ba98 fedc] k1'=[jihg] k2'=[nmlk] k3'=[po--] */
{
const int lshift = n*8-64, rshift = 32-lshift;
for( ; ptr < end ; ptr+=16) {
uint32_t c = k3>>rshift; // --98
k4 = READ_UINT32(ptr, 0); // dcba
c |= k4<<lshift; // ba98.
k3 = READ_UINT32(ptr, 1); // hgfe
k4 = k3<<lshift | k4>>rshift; // fedc.
doblock128x86(h1, h2, h3, h4, k1, k2, c, k4);
k2 = READ_UINT32(ptr, 2); // lkji
k1 = k2<<lshift | k3>>rshift; // jihg.
k3 = READ_UINT32(ptr, 3); // ponm.
k2 = k3<<lshift | k2>>rshift; // nmlk.
}
}
case 12: /*
k1=[3210] k2=[7654] k3=[ba98] k4=[----] w=[fedc jihg nmlk rqpo] b=[3210 7654 ba98 fedc] k1'=[jihg] k2'=[nmlk] k3'=[rqpo] */
for( ; ptr < end ; ptr+=16) {
k4 = READ_UINT32(ptr, 0);
doblock128x86(h1, h2, h3, h4, k1, k2, k3, k4);
k1 = READ_UINT32(ptr, 1);
k2 = READ_UINT32(ptr, 2);
k3 = READ_UINT32(ptr, 3);
}
break;
default: /* 12 < n <= 15
k1=[3210] k2=[7654] k3=[ba98] k4=[dc--] w=[hgfe lkji ponm tsrq] b=[3210 7654 ba98 fedc] k1'=[jihg] k2'=[nmlk] k3'=[rqpo] k3'=[ts--] */
{
const int lshift = n*8-96, rshift = 32-lshift;
for( ; ptr < end ; ptr+=16) {
uint32_t c = k4>>rshift; // --dc
k4 = READ_UINT32(ptr, 0); // hgfe
c |= k4<<lshift; // fedc.
doblock128x86(h1, h2, h3, h4, k1, k2, k3, c);
k3 = READ_UINT32(ptr, 1); // lkji
k1 = k3<<lshift | k4>>rshift; // jihg.
c = READ_UINT32(ptr, 2); // ponm
k2 = c<<lshift | k3>>rshift; // nmlk.
k4 = READ_UINT32(ptr, 3); // tsrq.
k3 = k4<<lshift | c>>rshift; // rqpo.
}
}
}
#endif /*UNALIGNED_SAFE*/
/* Advance over whole 128-bit chunks, possibly leaving 1..15 bytes */
len -= len & ~15;
/* Append any remaining bytes into carry */
dobytes128x86(len, h1, h2, h3, h4, k1, k2, k3, k4, n, ptr, len);
/* Copy out new running hash and carry */
ph[0] = h1;
ph[1] = h2;
ph[2] = h3;
ph[3] = h4;
pcarry[0] = k1;
pcarry[1] = k2;
pcarry[2] = k3;
pcarry[3] = (k4 & ~0xff) | n;
}
/*---------------------------------------------------------------------------*/
/* All in one go */
/* MurmurHash3_x86_128 api */
void PMurHash128x86(const void * key, const int len, uint32_t seed, void * out)
{
uint32_t carry[4] = {0, 0, 0, 0};
uint32_t h[4] = {seed, seed, seed, seed};
PMurHash128x86_Process(h, carry, key, len);
PMurHash128x86_Result(h, carry, (uint32_t) len, (uint32_t *) out);
}
/*-----------------------------------------------------------------------------*
PMurHash128x64
*-----------------------------------------------------------------------------*/
/*-----------------------------------------------------------------------------
* Core murmurhash algorithm macros */
static const uint64_t kC1L = BIG_CONSTANT(0x87c37b91114253d5);
static const uint64_t kC2L = BIG_CONSTANT(0x4cf5ad432745937f);
/* This is the main processing body of the algorithm. It operates
* on each full 128-bits of input. */
#define doblock128x64(h1, h2, k1, k2)\
do {\
k1 *= kC1L; k1 = ROTL64(k1,31); k1 *= kC2L; h1 ^= k1;\
\
h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;\
\
k2 *= kC2L; k2 = ROTL64(k2,33); k2 *= kC1L; h2 ^= k2;\
\
h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;\
} while(0)
/* Append unaligned bytes to carry, forcing hash churn if we have 16 bytes */
/* cnt=bytes to process, h1,h2=hash k1,k2=carry, n=bytes in carry, ptr/len=payload */
#define dobytes128x64(cnt, h1, h2, k1, k2, n, ptr, len) \
do {\
unsigned __cnt = cnt;\
for(;__cnt--; len--) {\
switch(n) {\
case 0: case 1: case 2: case 3:\
case 4: case 5: case 6: case 7:\
k1 = k1>>8 | (uint64_t)*ptr++<<56;\
n++; break;\
\
case 8: case 9: case 10: case 11:\
case 12: case 13: case 14:\
k2 = k2>>8 | (uint64_t)*ptr++<<56;\
n++; break;\
\
case 15:\
k2 = k2>>8 | (uint64_t)*ptr++<<56;\
doblock128x64(h1, h2, k1, k2);\
n = 0; break;\
}\
}\
} while(0)
/* Finalize a hash. To match the original Murmur3_128x64 the total_length must be provided */
void PMurHash128x64_Result(const uint64_t * const ph, const uint64_t * const pcarry,
const uint32_t total_length, uint64_t * const out)
{
uint64_t h1 = ph[0];
uint64_t h2 = ph[1];
uint64_t k1;
uint64_t k2 = pcarry[1];
int n = k2 & 15;
if (n) {
k1 = pcarry[0];
if (n > 8) {
k2 >>= (16-n)*8;
k2 *= kC2L; k2 = ROTL64(k2,33); k2 *= kC1L; h2 ^= k2;
} else {
k1 >>= (8-n)*8;
}
k1 *= kC1L; k1 = ROTL64(k1,31); k1 *= kC2L; h1 ^= k1;
}
//----------
// finalization
h1 ^= total_length; h2 ^= total_length;
h1 += h2;
h2 += h1;
h1 = fmix64(h1);
h2 = fmix64(h2);
h1 += h2;
h2 += h1;
out[0] = h1;
out[1] = h2;
}
/*---------------------------------------------------------------------------*/
/* Main hashing function. Initialise carry[2] to {0,0} and h[2] to an initial {seed,seed}
* if wanted. Both ph and pcarry are required arguments. */
void PMurHash128x64_Process(uint64_t * const ph, uint64_t * const pcarry, const void * const key, int len)
{
uint64_t h1 = ph[0];
uint64_t h2 = ph[1];
uint64_t k1 = pcarry[0];
uint64_t k2 = pcarry[1];
const uint8_t *ptr = (uint8_t*)key;
const uint8_t *end;
/* Extract carry count from low 4 bits of c value */
int n = k2 & 15;
#if defined(UNALIGNED_SAFE)
/* This CPU handles unaligned word access */
// #pragma message ( "UNALIGNED_SAFE" )
/* Consume any carry bytes */
int i = (16-n) & 15;
if(i && i <= len) {
dobytes128x64(i, h1, h2, k1, k2, n, ptr, len);
}
/* Process 128-bit chunks */
end = ptr + (len & ~15);
for( ; ptr < end ; ptr+=16) {
k1 = READ_UINT64(ptr, 0);
k2 = READ_UINT64(ptr, 1);
doblock128x64(h1, h2, k1, k2);
}
#else /*UNALIGNED_SAFE*/
/* This CPU does not handle unaligned word access */
// #pragma message ( "ALIGNED" )
/* Consume enough so that the next data byte is word aligned */
int i = -(intptr_t)(void *)ptr & 7;
if(i && i <= len) {
dobytes128x64(i, h1, h2, k1, k2, n, ptr, len);
}
/* We're now aligned. Process in aligned blocks. Specialise for each possible carry count */
end = ptr + (len & ~15);
switch(n) { /* how many bytes in c */
case 0: /*
k1=[--------] k2=[--------] w=[76543210 fedcba98] b=[76543210 fedcba98] */
for( ; ptr < end ; ptr+=16) {
k1 = READ_UINT64(ptr, 0);
k2 = READ_UINT64(ptr, 1);
doblock128x64(h1, h2, k1, k2);
}
break;
case 1: case 2: case 3: case 4: case 5: case 6: case 7: /*
k1=[10------] k2=[--------] w=[98765432 hgfedcba] b=[76543210 fedcba98] k1'=[hg------] */
{
const int lshift = n*8, rshift = 64-lshift;
for( ; ptr < end ; ptr+=16) {
uint64_t c = k1>>rshift;
k2 = READ_UINT64(ptr, 0);
c |= k2<<lshift;
k1 = READ_UINT64(ptr, 1);
k2 = k2>>rshift | k1<<lshift;
doblock128x64(h1, h2, c, k2);
}
}
break;
case 8: /*
k1=[76543210] k2=[--------] w=[fedcba98 nmlkjihg] b=[76543210 fedcba98] k1`=[nmlkjihg] */
for( ; ptr < end ; ptr+=16) {
k2 = READ_UINT64(ptr, 0);
doblock128x64(h1, h2, k1, k2);
k1 = READ_UINT64(ptr, 1);
}
break;
default: /* 8 < n <= 15
k1=[76543210] k2=[98------] w=[hgfedcba ponmlkji] b=[76543210 fedcba98] k1`=[nmlkjihg] k2`=[po------] */
{
const int lshift = n*8-64, rshift = 64-lshift;
for( ; ptr < end ; ptr+=16) {
uint64_t c = k2 >> rshift;
k2 = READ_UINT64(ptr, 0);
c |= k2 << lshift;
doblock128x64(h1, h2, k1, c);
k1 = k2 >> rshift;
k2 = READ_UINT64(ptr, 1);
k1 |= k2 << lshift;
}
}
}
#endif /*UNALIGNED_SAFE*/
/* Advance over whole 128-bit chunks, possibly leaving 1..15 bytes */
len -= len & ~15;
/* Append any remaining bytes into carry */
dobytes128x64(len, h1, h2, k1, k2, n, ptr, len);
/* Copy out new running hash and carry */
ph[0] = h1;
ph[1] = h2;
pcarry[0] = k1;
pcarry[1] = (k2 & ~0xff) | n;
}
/*---------------------------------------------------------------------------*/
/* All in one go */
/* MurmurHash3_x64_128 api */
void PMurHash128x64(const void * key, const int len, uint32_t seed, void * out)
{
uint64_t carry[2] = {0, 0};
uint64_t h[2] = {seed, seed};
PMurHash128x64_Process(h, carry, key, len);
PMurHash128x64_Result(h, carry, (uint32_t) len, (uint64_t *) out);
}

39
ext/hash/murmur/PMurHash128.h Normal file
View File

@@ -0,0 +1,39 @@
/*-----------------------------------------------------------------------------
* MurmurHash3 was written by Austin Appleby, and is placed in the public
* domain.
*
* This is a c++ implementation of MurmurHash3_128 with support for progressive
* processing based on PMurHash implementation written by Shane Day.
*/
/* ------------------------------------------------------------------------- */
// Microsoft Visual Studio
#if defined(_MSC_VER) && (_MSC_VER < 1600)
typedef unsigned char uint8_t;
typedef unsigned int uint32_t;
typedef unsigned __int64 uint64_t;
// Other compilers
#else // defined(_MSC_VER)
#include <stdint.h>
#endif // !defined(_MSC_VER)
/* ------------------------------------------------------------------------- */
/* Formal prototypes */
// PMurHash128x64
void PMurHash128x64_Process(uint64_t ph[2], uint64_t pcarry[2], const void *key, int len);
void PMurHash128x64_Result(const uint64_t ph[2], const uint64_t pcarry[2], uint32_t total_length, uint64_t out[2]);
void PMurHash128x64(const void * key, const int len, uint32_t seed, void * out);
// PMurHash128x86
void PMurHash128x86_Process(uint32_t ph[4], uint32_t pcarry[4], const void *key, int len);
void PMurHash128x86_Result(const uint32_t ph[4], const uint32_t pcarry[4], uint32_t total_length, uint32_t out[4]);
void PMurHash128x86(const void * key, const int len, uint32_t seed, void * out);

80
ext/hash/murmur/endianness.h Normal file
View File

@@ -0,0 +1,80 @@
static const union {
uint8_t u8[2];
uint16_t u16;
} EndianMix = {{ 1, 0 }};
FORCE_INLINE int IsBigEndian()
{
// Constant-folded by the compiler.
return EndianMix.u16 != 1;
}
#if defined(_MSC_VER)
# include <stdlib.h>
# define BSWAP32(u) _byteswap_ulong(u)
# define BSWAP64(u) _byteswap_uint64(u)
#else
# ifdef __has_builtin
# if __has_builtin(__builtin_bswap32)
# define BSWAP32(u) __builtin_bswap32(u)
# endif // __has_builtin(__builtin_bswap32)
# if __has_builtin(__builtin_bswap64)
# define BSWAP64(u) __builtin_bswap64(u)
# endif // __has_builtin(__builtin_bswap64)
# elif defined(__GNUC__) && ( \
__GNUC__ > 4 || ( \
__GNUC__ == 4 && ( \
__GNUC_MINOR__ >= 3 \
) \
) \
)
# define BSWAP32(u) __builtin_bswap32(u)
# define BSWAP64(u) __builtin_bswap64(u)
# endif // __has_builtin
#endif // defined(_MSC_VER)
#ifndef BSWAP32
FORCE_INLINE uint32_t BSWAP32(uint32_t u)
{
return (((u & 0xff000000) >> 24)
| ((u & 0x00ff0000) >> 8)
| ((u & 0x0000ff00) << 8)
| ((u & 0x000000ff) << 24));
}
#endif
#ifndef BSWAP64
FORCE_INLINE uint64_t BSWAP64(uint64_t u)
{
return (((u & 0xff00000000000000ULL) >> 56)
| ((u & 0x00ff000000000000ULL) >> 40)
| ((u & 0x0000ff0000000000ULL) >> 24)
| ((u & 0x000000ff00000000ULL) >> 8)
| ((u & 0x00000000ff000000ULL) << 8)
| ((u & 0x0000000000ff0000ULL) << 24)
| ((u & 0x000000000000ff00ULL) << 40)
| ((u & 0x00000000000000ffULL) << 56));
}
#endif
#ifdef __clang__
__attribute__((no_sanitize("alignment")))
#endif
FORCE_INLINE uint32_t getblock32 ( const uint32_t * const p, const int i)
{
if (IsBigEndian()) {
return BSWAP32(p[i]);
} else {
return p[i];
}
}
#ifdef __clang__
__attribute__((no_sanitize("alignment")))
#endif
FORCE_INLINE uint64_t getblock64 ( const uint64_t * const p, const int i)
{
if (IsBigEndian()) {
return BSWAP64(p[i]);
} else {
return p[i];
}
}

3
ext/hash/php_hash.h
View File

@@ -105,6 +105,9 @@ extern const php_hash_ops php_hash_fnv1a32_ops;
extern const php_hash_ops php_hash_fnv164_ops;
extern const php_hash_ops php_hash_fnv1a64_ops;
extern const php_hash_ops php_hash_joaat_ops;
extern const php_hash_ops php_hash_murmur3a_ops;
extern const php_hash_ops php_hash_murmur3c_ops;
extern const php_hash_ops php_hash_murmur3f_ops;
#define PHP_HASH_HAVAL_OPS(p,b) extern const php_hash_ops php_hash_##p##haval##b##_ops;

57
ext/hash/php_hash_murmur.h Normal file
View File

@@ -0,0 +1,57 @@
/*
+----------------------------------------------------------------------+
| Copyright (c) The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| http://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
| Author: Anatol Belski <ab@php.net> |
+----------------------------------------------------------------------+
*/
#ifndef PHP_HASH_MURMUR_H
#define PHP_HASH_MURMUR_H
typedef struct {
uint32_t h;
uint32_t carry;
uint32_t len;
} PHP_MURMUR3A_CTX;
#define PHP_MURMUR3A_SPEC "lll"
PHP_HASH_API void PHP_MURMUR3AInit(PHP_MURMUR3A_CTX *ctx);
PHP_HASH_API void PHP_MURMUR3AUpdate(PHP_MURMUR3A_CTX *ctx, const unsigned char *in, size_t len);
PHP_HASH_API void PHP_MURMUR3AFinal(unsigned char digest[4], PHP_MURMUR3A_CTX *ctx);
PHP_HASH_API int PHP_MURMUR3ACopy(const php_hash_ops *ops, PHP_MURMUR3A_CTX *orig_context, PHP_MURMUR3A_CTX *copy_context);
typedef struct {
uint32_t h[4];
uint32_t carry[4];
uint32_t len;
} PHP_MURMUR3C_CTX;
#define PHP_MURMUR3C_SPEC "lllllllll"
PHP_HASH_API void PHP_MURMUR3CInit(PHP_MURMUR3C_CTX *ctx);
PHP_HASH_API void PHP_MURMUR3CUpdate(PHP_MURMUR3C_CTX *ctx, const unsigned char *in, size_t len);
PHP_HASH_API void PHP_MURMUR3CFinal(unsigned char digest[16], PHP_MURMUR3C_CTX *ctx);
PHP_HASH_API int PHP_MURMUR3CCopy(const php_hash_ops *ops, PHP_MURMUR3C_CTX *orig_context, PHP_MURMUR3C_CTX *copy_context);
typedef struct {
uint64_t h[2];
uint64_t carry[2];
uint32_t len;
} PHP_MURMUR3F_CTX;
#define PHP_MURMUR3F_SPEC "qqqql"
PHP_HASH_API void PHP_MURMUR3FInit(PHP_MURMUR3F_CTX *ctx);
PHP_HASH_API void PHP_MURMUR3FUpdate(PHP_MURMUR3F_CTX *ctx, const unsigned char *in, size_t len);
PHP_HASH_API void PHP_MURMUR3FFinal(unsigned char digest[16], PHP_MURMUR3F_CTX *ctx);
PHP_HASH_API int PHP_MURMUR3FCopy(const php_hash_ops *ops, PHP_MURMUR3F_CTX *orig_context, PHP_MURMUR3F_CTX *copy_context);
#endif /* PHP_HASH_MURMUR_H */

18
ext/hash/tests/hash-clone.phpt
View File

@@ -143,6 +143,15 @@ string(16) "bebc746a33b6ab62"
string(5) "joaat"
string(8) "aaebf370"
string(8) "aaebf370"
string(8) "murmur3a"
string(8) "1b328135"
string(8) "1b328135"
string(8) "murmur3c"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(8) "murmur3f"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(10) "haval128,3"
string(32) "86362472c8895e68e223ef8b3711d8d9"
string(32) "86362472c8895e68e223ef8b3711d8d9"
@@ -302,6 +311,15 @@ string(16) "893899e4415a920f"
string(5) "joaat"
string(8) "aaebf370"
string(8) "836fb0e5"
string(8) "murmur3a"
string(8) "1b328135"
string(8) "18578d03"
string(8) "murmur3c"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(32) "2af4fdc002fda7b7491459e70377823f"
string(8) "murmur3f"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(32) "28249178bb182686ef793aa56abb6aea"
string(10) "haval128,3"
string(32) "86362472c8895e68e223ef8b3711d8d9"
string(32) "ebeeeb05c18af1e53d2d127b561d5e0d"

8
ext/hash/tests/hash_algos.phpt
View File

@@ -9,7 +9,7 @@ var_dump(hash_algos());
?>
--EXPECTF--
*** Testing hash_algos() : basic functionality ***
array(53) {
array(56) {
[%d]=>
string(3) "md2"
[%d]=>
@@ -87,6 +87,12 @@ array(53) {
[%d]=>
string(5) "joaat"
[%d]=>
string(8) "murmur3a"
[%d]=>
string(8) "murmur3c"
[%d]=>
string(8) "murmur3f"
[%d]=>
string(10) "haval128,3"
[%d]=>
string(10) "haval160,3"

18
ext/hash/tests/hash_copy_001.phpt
View File

@@ -143,6 +143,15 @@ string(16) "bebc746a33b6ab62"
string(5) "joaat"
string(8) "aaebf370"
string(8) "aaebf370"
string(8) "murmur3a"
string(8) "1b328135"
string(8) "1b328135"
string(8) "murmur3c"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(8) "murmur3f"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(10) "haval128,3"
string(32) "86362472c8895e68e223ef8b3711d8d9"
string(32) "86362472c8895e68e223ef8b3711d8d9"
@@ -302,6 +311,15 @@ string(16) "893899e4415a920f"
string(5) "joaat"
string(8) "aaebf370"
string(8) "836fb0e5"
string(8) "murmur3a"
string(8) "1b328135"
string(8) "18578d03"
string(8) "murmur3c"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(32) "2af4fdc002fda7b7491459e70377823f"
string(8) "murmur3f"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(32) "28249178bb182686ef793aa56abb6aea"
string(10) "haval128,3"
string(32) "86362472c8895e68e223ef8b3711d8d9"
string(32) "ebeeeb05c18af1e53d2d127b561d5e0d"

18
ext/hash/tests/hash_serialize_001.phpt
View File

@@ -154,6 +154,15 @@ string(16) "bebc746a33b6ab62"
string(5) "joaat"
string(8) "aaebf370"
string(8) "aaebf370"
string(8) "murmur3a"
string(8) "1b328135"
string(8) "1b328135"
string(8) "murmur3c"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(32) "2f041a2a310ba026921bc6ba34f17a2f"
string(8) "murmur3f"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(32) "aa86566cc6bf3a0987b83aabee30411e"
string(10) "haval128,3"
string(32) "86362472c8895e68e223ef8b3711d8d9"
string(32) "86362472c8895e68e223ef8b3711d8d9"
@@ -313,6 +322,15 @@ string(16) "893899e4415a920f"
string(5) "joaat"
string(8) "836fb0e5"
string(8) "836fb0e5"
string(8) "murmur3a"
string(8) "18578d03"
string(8) "18578d03"
string(8) "murmur3c"
string(32) "2af4fdc002fda7b7491459e70377823f"
string(32) "2af4fdc002fda7b7491459e70377823f"
string(8) "murmur3f"
string(32) "28249178bb182686ef793aa56abb6aea"
string(32) "28249178bb182686ef793aa56abb6aea"
string(10) "haval128,3"
string(32) "ebeeeb05c18af1e53d2d127b561d5e0d"
string(32) "ebeeeb05c18af1e53d2d127b561d5e0d"

2
ext/hash/tests/hash_serialize_002.phpt
View File

@@ -4,7 +4,7 @@ Hash: serialize()/unserialize() with HASH_HMAC
<?php
$algos = hash_algos();
$non_crypto = ["adler32", "crc32", "crc32b", "crc32c", "fnv132", "fnv1a32", "fnv164", "fnv1a64", "joaat"];
$non_crypto = ["adler32", "crc32", "crc32b", "crc32c", "fnv132", "fnv1a32", "fnv164", "fnv1a64", "joaat", "murmur3a", "murmur3c", "murmur3f"];
$key = "This is the key that I have";
foreach ($algos as $algo) {

49
ext/hash/tests/murmurhash3.phpt Normal file
View File

@@ -0,0 +1,49 @@
--TEST--
Hash: MurmurHash3 test
--FILE--
<?php
$h = hash("murmur3a", "foo");
echo $h, "\n";
$h = hash("murmur3c", "Two hashes meet in a bar", false);
echo $h, "\n";
$h = hash("murmur3c", "hash me!");
echo $h, "\n";
$h = hash("murmur3f", "Two hashes meet in a bar", false);
echo $h, "\n";
$h = hash("murmur3f", "hash me!");
echo $h, "\n";
$ctx = hash_init("murmur3a");
hash_update($ctx, "hello");
hash_update($ctx, " there");
$h0 = hash_final($ctx);
$h1 = hash("murmur3a", "hello there");
echo $h0, " ", $h1, "\n";
$ctx = hash_init("murmur3c");
hash_update($ctx, "hello");
hash_update($ctx, " there");
$h0 = hash_final($ctx);
$h1 = hash("murmur3c", "hello there");
echo $h0, " ", $h1, "\n";
$ctx = hash_init("murmur3f");
hash_update($ctx, "hello");
hash_update($ctx, " there");
$h0 = hash_final($ctx);
$h1 = hash("murmur3f", "hello there");
echo $h0, " ", $h1, "\n";
?>
--EXPECT--
f6a5c420
8036c2707453c6f37348142be7eaf75c
c7009299985a5627a9280372a9280372
40256ed26fa6ece7785092ed33c8b659
c43668294e89db0ba5772846e5804467
6440964d 6440964d
2bcadca212d62deb69712a721e593089 2bcadca212d62deb69712a721e593089
81514cc240f57a165c95eb63f9c0eedf 81514cc240f57a165c95eb63f9c0eedf