一致性hash的使用在PHP中有三种选择分别是原生的memcache扩展,memcached扩展,还有一个是网上比较流行的flexihash类。前两者都适用于memcache但不适合Redis。
php一致性hash类下载地址:http://code.google.com/p/flexihash/
我们根据flexihash的应该改写了一遍Redis的应用。
下面给出测试源码:
flexihash.php
flexihash_redis.php
FRedis_test.php
php一致性hash类下载地址:http://code.google.com/p/flexihash/
我们根据flexihash的应该改写了一遍Redis的应用。
下面给出测试源码:
flexihash.php
- <?php
- /**
- * Flexihash - A simple consistent hashing implementation for PHP.
- *
- * The MIT License
- *
- * Copyright (c) 2008 Paul Annesley
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- * @author Paul Annesley
- * @link http://paul.annesley.cc/
- * @copyright Paul Annesley, 2008
- * @comment by MyZ (http://blog.csdn.net/mayongzhan)
- */
- /**
- * A simple consistent hashing implementation with pluggable hash algorithms.
- *
- * @author Paul Annesley
- * @package Flexihash
- * @licence http://www.opensource.org/licenses/mit-license.php
- */
- class flexihash
- {
- /**
- * The number of positions to hash each target to.
- *
- * @var int
- * @comment 虚拟节点数,解决节点分布不均的问题
- */
- private $_replicas = 64;
- /**
- * The hash algorithm, encapsulated in a Flexihash_Hasher implementation.
- * @var object Flexihash_Hasher
- * @comment 使用的hash方法 : md5,crc32
- */
- private $_hasher;
- /**
- * Internal counter for current number of targets.
- * @var int
- * @comment 节点记数器
- */
- private $_targetCount = 0;
- /**
- * Internal map of positions (hash outputs) to targets
- * @var array { position => target, ... }
- * @comment 位置对应节点,用于lookup中根据位置确定要访问的节点
- */
- private $_positionToTarget = array();
- /**
- * Internal map of targets to lists of positions that target is hashed to.
- * @var array { target => [ position, position, ... ], ... }
- * @comment 节点对应位置,用于删除节点
- */
- private $_targetToPositions = array();
- /**
- * Whether the internal map of positions to targets is already sorted.
- * @var boolean
- * @comment 是否已排序
- */
- private $_positionToTargetSorted = false;
- /**
- * Constructor
- * @param object $hasher Flexihash_Hasher
- * @param int $replicas Amount of positions to hash each target to.
- * @comment 构造函数,确定要使用的hash方法和需拟节点数,虚拟节点数越多,分布越均匀,但程序的分布式运算越慢
- */
- public function __construct(Flexihash_Hasher $hasher = null, $replicas = null)
- {
- $this->_hasher = $hasher ? $hasher : new Flexihash_Crc32Hasher();
- if (!empty($replicas)) $this->_replicas = $replicas;
- }
- /**
- * Add a target.
- * @param string $target
- * @chainable
- * @comment 添加节点,根据虚拟节点数,将节点分布到多个虚拟位置上
- */
- public function addTarget($target)
- {
- if (isset($this->_targetToPositions[$target]))
- {
- throw new Flexihash_Exception("Target '$target' already exists.");
- }
- $this->_targetToPositions[$target] = array();
- // hash the target into multiple positions
- for ($i = 0; $i < $this->_replicas; $i++)
- {
- $position = $this->_hasher->hash($target . $i);
- $this->_positionToTarget[$position] = $target; // lookup
- $this->_targetToPositions[$target] []= $position; // target removal
- }
- $this->_positionToTargetSorted = false;
- $this->_targetCount++;
- return $this;
- }
- /**
- * Add a list of targets.
- * @param array $targets
- * @chainable
- */
- public function addTargets($targets)
- {
- foreach ($targets as $target)
- {
- $this->addTarget($target);
- }
- return $this;
- }
- /**
- * Remove a target.
- * @param string $target
- * @chainable
- */
- public function removeTarget($target)
- {
- if (!isset($this->_targetToPositions[$target]))
- {
- throw new Flexihash_Exception("Target '$target' does not exist.");
- }
- foreach ($this->_targetToPositions[$target] as $position)
- {
- unset($this->_positionToTarget[$position]);
- }
- unset($this->_targetToPositions[$target]);
- $this->_targetCount--;
- return $this;
- }
- /**
- * A list of all potential targets
- * @return array
- */
- public function getAllTargets()
- {
- return array_keys($this->_targetToPositions);
- }
- /**
- * Looks up the target for the given resource.
- * @param string $resource
- * @return string
- */
- public function lookup($resource)
- {
- $targets = $this->lookupList($resource, 1);
- if (empty($targets)) throw new Flexihash_Exception('No targets exist');
- return $targets[0];
- }
- /**
- * Get a list of targets for the resource, in order of precedence.
- * Up to $requestedCount targets are returned, less if there are fewer in total.
- *
- * @param string $resource
- * @param int $requestedCount The length of the list to return
- * @return array List of targets
- * @comment 查找当前的资源对应的节点,
- * 节点为空则返回空,节点只有一个则返回该节点,
- * 对当前资源进行hash,对所有的位置进行排序,在有序的位置列上寻找当前资源的位置
- * 当全部没有找到的时候,将资源的位置确定为有序位置的第一个(形成一个环)
- * 返回所找到的节点
- */
- public function lookupList($resource, $requestedCount)
- {
- if (!$requestedCount)
- throw new Flexihash_Exception('Invalid count requested');
- // handle no targets
- if (empty($this->_positionToTarget))
- return array();
- // optimize single target
- if ($this->_targetCount == 1)
- return array_unique(array_values($this->_positionToTarget));
- // hash resource to a position
- $resourcePosition = $this->_hasher->hash($resource);
- $results = array();
- $collect = false;
- $this->_sortPositionTargets();
- // search values above the resourcePosition
- foreach ($this->_positionToTarget as $key => $value)
- {
- // start collecting targets after passing resource position
- if (!$collect && $key > $resourcePosition)
- {
- $collect = true;
- }
- // only collect the first instance of any target
- if ($collect && !in_array($value, $results))
- {
- $results []= $value;
- }
- // return when enough results, or list exhausted
- if (count($results) == $requestedCount || count($results) == $this->_targetCount)
- {
- return $results;
- }
- }
- // loop to start - search values below the resourcePosition
- foreach ($this->_positionToTarget as $key => $value)
- {
- if (!in_array($value, $results))
- {
- $results []= $value;
- }
- // return when enough results, or list exhausted
- if (count($results) == $requestedCount || count($results) == $this->_targetCount)
- {
- return $results;
- }
- }
- // return results after iterating through both "parts"
- return $results;
- }
- public function __toString()
- {
- return sprintf(
- '%s{targets:[%s]}',
- get_class($this),
- implode(',', $this->getAllTargets())
- );
- }
- // ----------------------------------------
- // private methods
- /**
- * Sorts the internal mapping (positions to targets) by position
- */
- private function _sortPositionTargets()
- {
- // sort by key (position) if not already
- if (!$this->_positionToTargetSorted)
- {
- ksort($this->_positionToTarget, SORT_REGULAR);
- $this->_positionToTargetSorted = true;
- }
- }
- }
- /**
- * Hashes given values into a sortable fixed size address space.
- *
- * @author Paul Annesley
- * @package Flexihash
- * @licence http://www.opensource.org/licenses/mit-license.php
- */
- interface Flexihash_Hasher
- {
- /**
- * Hashes the given string into a 32bit address space.
- *
- * Note that the output may be more than 32bits of raw data, for example
- * hexidecimal characters representing a 32bit value.
- *
- * The data must have 0xFFFFFFFF possible values, and be sortable by
- * PHP sort functions using SORT_REGULAR.
- *
- * @param string
- * @return mixed A sortable format with 0xFFFFFFFF possible values
- */
- public function hash($string);
- }
- /**
- * Uses CRC32 to hash a value into a signed 32bit int address space.
- * Under 32bit PHP this (safely) overflows into negatives ints.
- *
- * @author Paul Annesley
- * @package Flexihash
- * @licence http://www.opensource.org/licenses/mit-license.php
- */
- class Flexihash_Crc32Hasher
- implements Flexihash_Hasher
- {
- /* (non-phpdoc)
- * @see Flexihash_Hasher::hash()
- */
- public function hash($string)
- {
- return crc32($string);
- }
- }
- /**
- * Uses CRC32 to hash a value into a 32bit binary string data address space.
- *
- * @author Paul Annesley
- * @package Flexihash
- * @licence http://www.opensource.org/licenses/mit-license.php
- */
- class Flexihash_Md5Hasher
- implements Flexihash_Hasher
- {
- /* (non-phpdoc)
- * @see Flexihash_Hasher::hash()
- */
- public function hash($string)
- {
- return substr(md5($string), 0, 8); // 8 hexits = 32bit
- // 4 bytes of binary md5 data could also be used, but
- // performance seems to be the same.
- }
- }
- /**
- * An exception thrown by Flexihash.
- *
- * @author Paul Annesley
- * @package Flexihash
- * @licence http://www.opensource.org/licenses/mit-license.php
- */
- class Flexihash_Exception extends Exception
- {
- }
flexihash_redis.php
- <?php
- require 'flexihash.php';
- Class FRedis
- {
- public $hash = null;
- public $memcache = null;
- public $connectPool = null;
- public $timeOut = 3;
- public $auth = "";
- public function __construct()
- {
- $this->hash = new Flexihash();
- }
- public function addServers( $servers )
- {
- foreach ($servers as $server)
- {
- $node = $server['host'] . ':' . $server['port'];
- $this->connectPool[$node] = false;
- $targets[] = $node;
- }
- $this->hash->addTargets( $targets );
- }
- public function hset($hashkey, $valuekey, $value )
- {
- $node = $this->hash->lookup( $hashkey, count( $this->connectPool ) );
- if (!$this->connectPool[$node])
- {
- $server = explode( ':', $node );
- $redis = new Redis();
- $redis->connect($server[0], $server[1], $this->timeOut);
- if(!empty($this->auth)){
- $redis->auth($this->auth);
- }
- $this->connectPool[$node] = $redis;
- }
- if ($this->connectPool[$node])
- {
- $redis = $this->connectPool[$node];
- if($redis->hset($hashkey,$valuekey,$value)){
- return true;
- }
- }
- return false;
- }
- public function hget( $hashkey, $valuekey )
- {
- $node = $this->hash->lookup( $hashkey, count( $this->connectPool ) );
- if (!$this->connectPool[$node])
- {
- $server = explode( ':', $node );
- $redis = new Redis();
- $redis->connect($server[0], $server[1], $this->timeOut);
- if(!empty($this->auth)){
- $redis->auth($this->auth);
- }
- $this->connectPool[$node] = $redis;
- }
- if ($this->connectPool[$node])
- {
- $redis = $this->connectPool[$node];
- if($value = $redis->hget($hashkey,$valuekey)){
- return $value;
- }
- }
- return false;
- }
- }
FRedis_test.php
- <?php
- require_once 'flexihash_redis.php';
- $st = microtime( true );
- $fredis = new FRedis();
- $tt_server_pool = array(
- '0' => array(
- 'host' => '192.168.75.128',
- 'port' => '6379',
- ),
- '1' => array(
- 'host' => '192.168.75.130',
- 'port' => '6379',
- ),
- );
- $fredis->addServers( $tt_server_pool );
- for($i=0;$i<10;$i++){
- for($k=0;$k<10;$k++){
- $fredis->hset("hashkey:".$i,"valuekey".$k,md5($i*$k));
- }
- }
- $et = microtime( true ) - $st;
- echo "time used:" . $et."<br/>";
- echo $fredis->hget("hashkey:0","valuekey9");
Redis版哈希函数类:
- <?php
- class UniqueHashService{
- private static $len; //64位
- private static $loop = 1; //循环次数
- private $prefix = '';
- public function __construct() {
- if(empty(self::$len)){
- self::$len = abs(1 << 0x3f); //左移63位
- }
- }
- /**
- * 获取字符串的哈希值
- */
- public function hashing($key){
- return abs((('0x'.hash('crc32',$key).hash('crc32b',$key)) + 0)) % self::$len;
- }
- /**
- * 获取到redis hash 的键名
- * @param type $str
- */
- public function getRedisKey($str, $prefix){
- //前缀
- $this->prefix = $prefix;
- $hstr = $this->hashing($str); //获取一个字符串的哈希值
- //如果字符串长度小于6,往字符串左侧填充0,填充至6位长度。
- if(strlen($hstr) < 6){
- $hstr = str_pad($hstr, 6, '0', STR_PAD_LEFT);
- }
- $kbin = $this->str2Bin($hstr); //计算hash key的二进制,$hstr的二进制
- $karr = explode(' ', $kbin);
- //数组长度
- $kcount = count($karr);
- $kcount_mid = ceil($kcount/2);
- $redis_key = $this->prefix; //排重表前缀
- //循环次数
- $i = 0;
- while(1){
- if($i > self::$loop){
- break;
- }
- //组装redis键
- $redis_key .= ':'.$this->returnHashKey($karr[$i]);
- $redis_key .= ':'.$this->returnHashKey($karr[$kcount_mid+$i]);
- $redis_key .= ':'.$this->returnHashKey($karr[$kcount-1-$i]);
- ++$i;
- }
- return $redis_key;
- }
- /**
- * 截取二进制字符串中的一部分
- * @param type $sbin
- * @return type
- */
- private function returnHashKey($sbin){
- return substr($sbin, -2, 2);
- }
- /**
- * 把给定的字符串中的每个字符转为二进制,并以空格连接每个二进制字符串并返回。
- * @param string $hstr
- * @return type
- */
- private function str2Bin($str){
- //把字符串拆分成数组,每个字符为数组中的一个元素。
- $arr = str_split($str);
- //把数组中的每个字符转成二进制
- foreach($arr as &$v){
- $tmp = unpack('H*', $v); //把字符解包为16进制
- $v = base_convert($tmp[1],16,2);//把16进制转为2进制
- unset($tmp);
- }
- //把二进制字符串数组以空格连接在一起并返回
- $bin = join(' ',$arr);
- unset($arr);
- return $bin;
- }
- /**
- * 把$this->str2Bin 处理过的字符串还原
- * @param string $bin 二进制字符串
- * @return type
- */
- private function bin2Str($bin){
- //把字符串拆分成数组,以空拆分。
- $arr = explode(' ', $bin);
- foreach($arr as &$v){
- $tmp = base_convert($v, 2, 16); //二进制转换成16进制
- $v = pack('H'.strlen($tmp), $tmp); //把16进制打包成二进制字符串
- unset($tmp);
- }
- return join('', $arr);
- }
- }